From 0a59aec1ba38a528ffabf90d7dd2ad245d76d581 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:37:40 +0200 Subject: [PATCH 01/12] perf(screenshot): Avoid redundant buffers and copies in screenshot uploads --- .../GeneralsOnline/OnlineServices_Init.h | 4 +- .../GeneralsOnline/HTTP/HTTPManager.cpp | 2 +- .../GeneralsOnline/OnlineServices_Init.cpp | 203 ++++++++---------- 3 files changed, 88 insertions(+), 121 deletions(-) diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index 7d1419d2ee1..27d5fffdfcd 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -586,10 +586,10 @@ class NGMP_OnlineServicesManager ServiceConfig& GetServiceConfig() { return m_ServiceConfig; } public: - void CacheScreenshotBytes_StartMatch(std::vector& vecData) + void CacheScreenshotBytes_StartMatch(std::vector vecData) { std::scoped_lock ssLock(m_ScreenshotMutex); - m_vecCachedScreenshotBytes_MatchStart = vecData; + m_vecCachedScreenshotBytes_MatchStart = std::move(vecData); } void CacheScreenshotBytes_EndMatch(uint64_t matchID, std::vector data); diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp index a420a04cda6..9b6d94eca4c 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp @@ -49,7 +49,7 @@ void HTTPManager::SendS3PUTRequest(const char* szURI, EIPProtocolVersion protove HTTPRequest* pRequest = PlatformCreateRequest(EHTTPVerb::HTTP_VERB_PUT, protover, szURI, inHeaders, completionCallback, progressCallback, timeoutMS); pRequest->DisableServiceAuth(); - pRequest->SetPostDataBuffer(vecBuffer); + pRequest->SetPostDataBuffer(std::move(vecBuffer)); m_vecRequestsPendingStart.push_back(pRequest); } diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp index cfe0835efd4..951c40f6dd2 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp @@ -127,7 +127,7 @@ void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType scree // certain screenshots require caching for later upload when we have a valid match id, so we store them if (screenshotType == EScreenshotType::SCREENSHOT_TYPE_LOADSCREEN) { - NGMP_OnlineServicesManager::GetInstance()->CacheScreenshotBytes_StartMatch(vecData); + NGMP_OnlineServicesManager::GetInstance()->CacheScreenshotBytes_StartMatch(std::move(vecData)); } else if (screenshotType == EScreenshotType::SCREENSHOT_TYPE_SCORESCREEN) { @@ -143,7 +143,7 @@ void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType scree newEntry.vecBytes = std::move(vecData); newEntry.strSignedURI = strURI; newEntry.screenshotType = screenshotType; - m_vecGuardedSSData.push_back(newEntry); + m_vecGuardedSSData.push_back(std::move(newEntry)); } }); } @@ -579,7 +579,6 @@ void NGMP_OnlineServicesManager::CaptureScreenshot(bool bResizeForTransmit, std: } SurfaceClass* surface = DX8Wrapper::_Get_DX8_Back_Buffer(); - LPDIRECT3DSURFACE8 surf = nullptr; SurfaceClass* surfaceCopy = nullptr; void* pBits = nullptr; IDirect3DSurface8* pDXsurf = nullptr; @@ -599,119 +598,96 @@ void NGMP_OnlineServicesManager::CaptureScreenshot(bool bResizeForTransmit, std: { DX8Wrapper::_Copy_DX8_Rects(surface->Peek_D3D_Surface(), NULL, 0, surfaceCopy->Peek_D3D_Surface(), NULL); - HRESULT hr; + // gather all our data + int pitch = 0; + pBits = surfaceCopy->Lock(&pitch); - D3DDISPLAYMODE mode; - if (SUCCEEDED(hr = DX8Wrapper::_Get_D3D_Device8()->GetDisplayMode(&mode))) + if (pBits != nullptr) { - if (SUCCEEDED(hr = DX8Wrapper::_Get_D3D_Device8()->CreateImageSurface(mode.Width, mode.Height, - D3DFMT_A8R8G8B8, &surf))) - { - if (SUCCEEDED(hr = DX8Wrapper::_Get_D3D_Device8()->GetFrontBuffer(surf))) + int width = surfaceDesc.Width; + int height = surfaceDesc.Height; + + // Copy pixel data into an owned buffer before spawning the thread so + // the surface can be safely unlocked on the main thread immediately after. + const uint8_t* pSurfaceBytes = static_cast(pBits); + std::vector pixelData(pSurfaceBytes, pSurfaceBytes + height * pitch); + + // process on thread - track the thread so we can join it during shutdown + std::thread* pNewThread = new std::thread([cbOnDataAvailable, width, height, pixelData = std::move(pixelData), pitch, bResizeForTransmit]() mutable { - // gather all our data - int pitch = 0; - pBits = surfaceCopy->Lock(&pitch); + CHECK_WORKER_THREAD; - if (pBits != nullptr) - { - int width = surfaceDesc.Width; - int height = surfaceDesc.Height; - - // Copy pixel data into an owned buffer before spawning the thread so - // the surface can be safely unlocked on the main thread immediately after. - std::vector pixelData(height * pitch); - memcpy(pixelData.data(), pBits, height * pitch); - - // process on thread - track the thread so we can join it during shutdown - std::thread* pNewThread = new std::thread([cbOnDataAvailable, width, height, pixelData = std::move(pixelData), pitch, bResizeForTransmit]() - { - CHECK_WORKER_THREAD; - - unsigned char* rgbData = new unsigned char[width * height * 3]; - - std::vector vecData; - - int finalWidth = width; - int finalHeight = height; - - for (int y = 0; y < height; ++y) { - const uint8_t* row = pixelData.data() + y * pitch; - int rowOffset = y * width * 3; - int srcOffset = 0; - for (int x = 0; x < width; ++x, srcOffset += 4) - { - int dstIndex = rowOffset + x * 3; - rgbData[dstIndex + 0] = row[srcOffset + 2]; // R - rgbData[dstIndex + 1] = row[srcOffset + 1]; // G - rgbData[dstIndex + 2] = row[srcOffset + 0]; // B - } - } - - // resize - unsigned char* pBufferToWrite = rgbData; - if (bResizeForTransmit) - { - ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - int new_width = serviceConf.screenshot_width; - int new_height = serviceConf.screenshot_height; - int channels = 3; - unsigned char* resized = new unsigned char[new_width * new_height * channels]; - - stbir_resize_uint8(rgbData, width, height, 0, - resized, new_width, new_height, 0, - channels - ); - - // update data - finalWidth = new_width; - finalHeight = new_height; - pBufferToWrite = resized; - } - // end resize - - stbi_write_jpg_to_func([](void* context, void* data, int size) - { - std::vector* buffer = static_cast*>(context); - buffer->insert(buffer->end(), (unsigned char*)data, (unsigned char*)data + size); - }, &vecData, finalWidth, finalHeight, 3, pBufferToWrite, bResizeForTransmit ? 0 : 90); - - // cleanup - if (bResizeForTransmit) - { - delete[] pBufferToWrite; // This is 'resized' - pBufferToWrite = nullptr; - } - - delete[] rgbData; - rgbData = nullptr; - - // invoke cb - if (cbOnDataAvailable != nullptr) - { - cbOnDataAvailable(vecData); - } - } - ); + const int channels = 3; + int finalWidth = width; + int finalHeight = height; - // Store the thread so we can join it during shutdown - // SECURITY FIX: Capture manager pointer before spawning thread to avoid TOCTOU race - NGMP_OnlineServicesManager* pMgr = NGMP_OnlineServicesManager::GetInstance(); - if (pMgr != nullptr) + // Pack the BGRA rows to tight RGB in place. The write position never passes the read position. + uint8_t* pixels = pixelData.data(); + for (int y = 0; y < height; ++y) { - std::scoped_lock lock(pMgr->m_mutexScreenshotThreads); - pMgr->m_vecScreenshotThreads.push_back(pNewThread); + const uint8_t* src = pixels + y * pitch; + uint8_t* dst = pixels + y * width * channels; + for (int x = 0; x < width; ++x, src += 4, dst += channels) + { + const uint8_t blue = src[0]; + dst[0] = src[2]; // R + dst[1] = src[1]; // G + dst[2] = blue; // B + } } - else + + // resize + std::vector resized; + if (bResizeForTransmit) { - // Manager was destroyed, cannot store thread. Thread will leak but won't crash. - NetworkLog(ELogVerbosity::LOG_RELEASE, "[Screenshot] Manager destroyed before thread could be registered"); + ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); + finalWidth = serviceConf.screenshot_width; + finalHeight = serviceConf.screenshot_height; + resized.resize(finalWidth * finalHeight * channels); + + stbir_resize_uint8(pixels, width, height, 0, + resized.data(), finalWidth, finalHeight, 0, + channels + ); + + // the capture is no longer needed + std::vector().swap(pixelData); } + // end resize + + // room for a typical JPEG, so the write callback rarely has to grow it + std::vector vecData; + vecData.reserve(256 * 1024); + + stbi_write_jpg_to_func([](void* context, void* data, int size) + { + std::vector* buffer = static_cast*>(context); + buffer->insert(buffer->end(), (unsigned char*)data, (unsigned char*)data + size); + }, &vecData, finalWidth, finalHeight, channels, bResizeForTransmit ? resized.data() : pixelData.data(), bResizeForTransmit ? 0 : 90); - bSucceeded = true; + // invoke cb + if (cbOnDataAvailable != nullptr) + { + cbOnDataAvailable(std::move(vecData)); } } + ); + + // Store the thread so we can join it during shutdown + // SECURITY FIX: Capture manager pointer before spawning thread to avoid TOCTOU race + NGMP_OnlineServicesManager* pMgr = NGMP_OnlineServicesManager::GetInstance(); + if (pMgr != nullptr) + { + std::scoped_lock lock(pMgr->m_mutexScreenshotThreads); + pMgr->m_vecScreenshotThreads.push_back(pNewThread); + } + else + { + // Manager was destroyed, cannot store thread. Thread will leak but won't crash. + NetworkLog(ELogVerbosity::LOG_RELEASE, "[Screenshot] Manager destroyed before thread could be registered"); } + + bSucceeded = true; } } } @@ -719,18 +695,9 @@ void NGMP_OnlineServicesManager::CaptureScreenshot(bool bResizeForTransmit, std: // clean everything up, whether we succeeded or not - // release the image surface - if (surf != nullptr) - { - surf->Release(); - //delete surf; - surf = nullptr; - } - // unlock if (surface != nullptr) { - surface->Unlock(); surface->Release_Ref(); surface = nullptr; } @@ -954,7 +921,7 @@ void NGMP_OnlineServicesManager::Tick() // screenshot types that already have a presigned URL - for (S3ScreenshotEntry screenshotEntry : m_vecGuardedSSData) + for (S3ScreenshotEntry& screenshotEntry : m_vecGuardedSSData) { // NOTE: Screenshot types start and end of match are captures and cached in memory until the server tells us where to upload them, so we need to wait for the upload URI @@ -966,7 +933,7 @@ void NGMP_OnlineServicesManager::Tick() { std::map mapHeaders; mapHeaders["Content-Type"] = "image/jpeg"; - NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()->SendS3PUTRequest(screenshotEntry.strSignedURI.c_str(), EIPProtocolVersion::DONT_CARE, mapHeaders, screenshotEntry.vecBytes, [=](bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq) + NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()->SendS3PUTRequest(screenshotEntry.strSignedURI.c_str(), EIPProtocolVersion::DONT_CARE, mapHeaders, std::move(screenshotEntry.vecBytes), [=](bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq) { #if _DEBUG if (statusCode != 200) @@ -990,9 +957,9 @@ void NGMP_OnlineServicesManager::Tick() // queue it S3ScreenshotEntry newEntry; newEntry.screenshotType = EScreenshotType::SCREENSHOT_TYPE_LOADSCREEN; - newEntry.vecBytes = m_vecCachedScreenshotBytes_MatchStart; - newEntry.strSignedURI = m_strCachedScreenshot_MatchStart_S3URI; - m_vecGuardedSSData.push_back(newEntry); + newEntry.vecBytes = std::move(m_vecCachedScreenshotBytes_MatchStart); + newEntry.strSignedURI = std::move(m_strCachedScreenshot_MatchStart_S3URI); + m_vecGuardedSSData.push_back(std::move(newEntry)); // clear data m_vecCachedScreenshotBytes_MatchStart = std::vector(); @@ -1023,7 +990,7 @@ void NGMP_OnlineServicesManager::Tick() // do the upload std::map mapHeaders; mapHeaders["Content-Type"] = "application/octet-stream"; - NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()->SendS3PUTRequest(m_cachedReplayUpload.signedURI.c_str(), EIPProtocolVersion::DONT_CARE, mapHeaders, m_cachedReplayUpload.bytes, [=](bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq) + NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()->SendS3PUTRequest(m_cachedReplayUpload.signedURI.c_str(), EIPProtocolVersion::DONT_CARE, mapHeaders, std::move(m_cachedReplayUpload.bytes), [=](bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq) { #if _DEBUG if (statusCode != 200) From 4e21dabadfc6e22df9d19667e2bc7f9d05e22e6d Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:32:41 +0100 Subject: [PATCH 02/12] perf(screenshot): Upload probe screenshots one slot at a time at a capped rate --- .../GeneralsOnline/HTTP/HTTPManager.h | 2 +- .../GeneralsOnline/HTTP/HTTPRequest.h | 7 +++ .../GeneralsOnline/OnlineServices_Init.h | 6 +- .../GeneralsOnline/HTTP/HTTPManager.cpp | 3 +- .../GeneralsOnline/HTTP/HTTPRequest.cpp | 5 ++ .../GeneralsOnline/OnlineServices_Init.cpp | 60 +++++++++++++++++-- 6 files changed, 74 insertions(+), 9 deletions(-) diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h index 57a4c498928..d2d5c80f753 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h @@ -54,7 +54,7 @@ class HTTPManager // NOTE: set bDisableServiceAuth for endpoints that must be called with something other than the session token (e.g. the refresh endpoint, which needs the refresh token), otherwise the session token overwrites any Authorization header passed in inHeaders void SendPOSTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, const char* szPostData, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1, bool bDisableServiceAuth = false); void SendPUTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, const char* szData, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1); - void SendS3PUTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, std::vector vecBuffer, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1); + void SendS3PUTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, std::vector vecBuffer, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1, int maxSendBytesPerSec = 0); void SendDELETERequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, const char* szData, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1); void Shutdown(); diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h index f5b901aade5..7dbf977a136 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h @@ -36,6 +36,11 @@ class HTTPRequest m_bAppendAuthIfPresent = false; } + void SetMaxSendSpeed(int bytesPerSec) + { + m_maxSendBytesPerSec = bytesPerSec; + } + void OnResponsePartialWrite(std::uint8_t* pBuffer, size_t numBytes); bool HasStarted() const { return m_bIsStarted; } @@ -77,6 +82,8 @@ class HTTPRequest bool m_bAppendAuthIfPresent = true; + int m_maxSendBytesPerSec = 0; // 0 = unlimited + EIPProtocolVersion m_protover; int m_timeoutMS = 5000; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index 27d5fffdfcd..2096e7b3623 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -4,6 +4,7 @@ #include #include +#include class HTTPManager; @@ -28,6 +29,7 @@ struct S3ScreenshotEntry std::vector vecBytes; std::string strSignedURI; EScreenshotType screenshotType; + std::chrono::steady_clock::time_point notBefore; // upload is held back until this time bool operator==(const S3ScreenshotEntry& other) const { @@ -341,6 +343,8 @@ struct ServiceConfig int screenshot_width = 557; int screenshot_height = 333; + int screenshot_upload_bytes_per_sec = 131072; // 128 KB/s + int screenshot_slot_offset_ms = 2000; // ICE client used for P2P NAT traversal: 0 = library default, 1 = native, 2 = WebRTC. int ice_implementation = 2; @@ -351,7 +355,7 @@ struct ServiceConfig ra_slack_percent, frame_grouping_frames, enable_host_migration, network_do_immediate_flush_per_frame, network_send_flags, network_latency_logic_model, use_default_config, ra_slack_override_percent_in_default, do_probes, do_replay_upload, network_mesh_histogram_duration, ibra_ra_tweaks, ibra_minslack_default, ibra_maxslack_default, ibra_minslack_greaterthan300ms, ibra_maxslack_greaterthan300ms, ibra_minslack_greaterthan200ms, ibra_maxslack_greaterthan200ms, - screenshot_width, screenshot_height, ice_implementation, use_new_networking) + screenshot_width, screenshot_height, screenshot_upload_bytes_per_sec, screenshot_slot_offset_ms, ice_implementation, use_new_networking) }; class NGMP_OnlineServicesManager diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp index 9b6d94eca4c..e476ced3bb7 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp @@ -43,12 +43,13 @@ void HTTPManager::SendPUTRequest(const char* szURI, EIPProtocolVersion protover, } -void HTTPManager::SendS3PUTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, std::vector vecBuffer, std::function completionCallback, std::function progressCallback /*= nullptr*/, int timeoutMS /*= -1*/) +void HTTPManager::SendS3PUTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, std::vector vecBuffer, std::function completionCallback, std::function progressCallback /*= nullptr*/, int timeoutMS /*= -1*/, int maxSendBytesPerSec /*= 0*/) { CHECK_MAIN_THREAD; HTTPRequest* pRequest = PlatformCreateRequest(EHTTPVerb::HTTP_VERB_PUT, protover, szURI, inHeaders, completionCallback, progressCallback, timeoutMS); pRequest->DisableServiceAuth(); + pRequest->SetMaxSendSpeed(maxSendBytesPerSec); pRequest->SetPostDataBuffer(std::move(vecBuffer)); m_vecRequestsPendingStart.push_back(pRequest); diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp index 02786d7f3d9..3c77e8f4039 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp @@ -298,6 +298,11 @@ void HTTPRequest::PlatformStartRequest() } } + if (m_maxSendBytesPerSec > 0) + { + curl_easy_setopt(m_pCURL, CURLOPT_MAX_SEND_SPEED_LARGE, (curl_off_t)m_maxSendBytesPerSec); + } + // needed for PUT etc if (m_httpVerb == EHTTPVerb::HTTP_VERB_PUT) { diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp index 951c40f6dd2..b2fa378fdb1 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp @@ -23,7 +23,10 @@ #include "GameNetwork/GeneralsOnline/Vendor/stb_image/stb_image_write.h" #include "GameNetwork/GeneralsOnline/Vendor/stb_image/stb_image_resize.h" #include "GameClient/GameText.h" +#include "GameLogic/GameLogic.h" #include +#include +#include extern "C" { @@ -99,6 +102,34 @@ void NGMP_OnlineServicesManager::GetAndParseServiceConfig(std::function(); + NGMP_OnlineServices_LobbyInterface* pLobbyInterface = NGMP_OnlineServicesManager::GetInterface(); + if (pAuthInterface != nullptr && pLobbyInterface != nullptr) + { + const int64_t localUserID = pAuthInterface->GetUserID(); + for (const LobbyMemberEntry& lobbyMember : pLobbyInterface->GetMembersListForCurrentRoom()) + { + if (lobbyMember.user_id == localUserID) + { + slotIndex = lobbyMember.m_SlotIndex < MAX_SLOTS ? lobbyMember.m_SlotIndex : 0; + break; + } + } + } + + const int slotOffsetMS = std::clamp(NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig().screenshot_slot_offset_ms, 0, 10000); + + static thread_local std::mt19937 rng{ std::random_device{}() }; + const int jitterMS = std::uniform_int_distribution(0, 500)(rng); + + return std::chrono::steady_clock::now() + std::chrono::milliseconds(slotIndex * slotOffsetMS + jitterMS); +} + void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType screenshotType, std::string strURI) { NGMP_OnlineServicesManager* pOnlineServicesMgr = NGMP_OnlineServicesManager::GetInstance(); @@ -114,7 +145,8 @@ void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType scree if (pLobbyInterface != nullptr) { const uint64_t matchID = pLobbyInterface->GetCurrentMatchID(); - NGMP_OnlineServicesManager::GetInstance()->CaptureScreenshot(true, [strURI = std::move(strURI), screenshotType, matchID](std::vector vecData) + const std::chrono::steady_clock::time_point uploadTime = GetScreenshotUploadTime(); + NGMP_OnlineServicesManager::GetInstance()->CaptureScreenshot(true, [strURI = std::move(strURI), screenshotType, matchID, uploadTime](std::vector vecData) { CHECK_WORKER_THREAD; @@ -143,6 +175,7 @@ void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType scree newEntry.vecBytes = std::move(vecData); newEntry.strSignedURI = strURI; newEntry.screenshotType = screenshotType; + newEntry.notBefore = uploadTime; m_vecGuardedSSData.push_back(std::move(newEntry)); } }); @@ -920,13 +953,28 @@ void NGMP_OnlineServicesManager::Tick() std::scoped_lock ssLock(m_ScreenshotMutex); + const int uploadBytesPerSec = std::clamp(m_ServiceConfig.screenshot_upload_bytes_per_sec, 16384, 10485760); + const std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now(); + const bool bMatchOver = TheGameLogic == nullptr || !TheGameLogic->isInMultiplayerGame(); // uploads no longer compete with game traffic + // screenshot types that already have a presigned URL - for (S3ScreenshotEntry& screenshotEntry : m_vecGuardedSSData) + for (auto it = m_vecGuardedSSData.begin(); it != m_vecGuardedSSData.end();) { + S3ScreenshotEntry& screenshotEntry = *it; + const bool bUnthrottled = bMatchOver || screenshotEntry.screenshotType == EScreenshotType::SCREENSHOT_TYPE_SCORESCREEN; + + // not this client's turn to upload yet + if (!bUnthrottled && screenshotEntry.notBefore > now) + { + ++it; + continue; + } + // NOTE: Screenshot types start and end of match are captures and cached in memory until the server tells us where to upload them, so we need to wait for the upload URI if ((screenshotEntry.screenshotType == EScreenshotType::SCREENSHOT_TYPE_LOADSCREEN || screenshotEntry.screenshotType == EScreenshotType::SCREENSHOT_TYPE_SCORESCREEN) && screenshotEntry.strSignedURI.empty()) { + it = m_vecGuardedSSData.erase(it); continue; } else // we have the data we need, send away @@ -942,12 +990,11 @@ void NGMP_OnlineServicesManager::Tick() } #endif NetworkLog(ELogVerbosity::LOG_RELEASE, "Screenshot upload, result: %d", statusCode); - }, nullptr, HTTP_UPLOAD_TIMEOUT); + }, nullptr, HTTP_UPLOAD_TIMEOUT, bUnthrottled ? 0 : uploadBytesPerSec); } - - } - m_vecGuardedSSData.clear(); + it = m_vecGuardedSSData.erase(it); + } // screenshots and replays that are cached and awaiting S3 url, and now have said URL if (!m_vecCachedScreenshotBytes_MatchStart.empty()) // we have data waiting @@ -959,6 +1006,7 @@ void NGMP_OnlineServicesManager::Tick() newEntry.screenshotType = EScreenshotType::SCREENSHOT_TYPE_LOADSCREEN; newEntry.vecBytes = std::move(m_vecCachedScreenshotBytes_MatchStart); newEntry.strSignedURI = std::move(m_strCachedScreenshot_MatchStart_S3URI); + newEntry.notBefore = GetScreenshotUploadTime(); m_vecGuardedSSData.push_back(std::move(newEntry)); // clear data From 3580bf4bf28a4c404b26cefe46e634a79f7ddbe8 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Thu, 22 Oct 2026 06:59:06 +0200 Subject: [PATCH 03/12] feat(network): Share the game network settings between LAN and online --- .../Include/GameNetwork/ConnectionManager.h | 8 + .../Include/GameNetwork/FrameMetrics.h | 3 + .../Include/GameNetwork/NetworkDefs.h | 2 +- .../Include/GameNetwork/NetworkSettings.h | 35 ++++ .../Source/GameNetwork/Connection.cpp | 2 +- .../Source/GameNetwork/ConnectionManager.cpp | 29 ++- .../Source/GameNetwork/DisconnectManager.cpp | 2 +- .../Source/GameNetwork/FrameMetrics.cpp | 30 ++- .../GeneralsOnline/NextGenMP_defines.h | 6 +- .../GeneralsOnline/OnlineServices_Init.h | 190 ++++++++++++++---- .../GUICallbacks/Menus/WOLGameSetupMenu.cpp | 2 +- .../GameNetwork/GeneralsOnline/NGMPGame.cpp | 34 +--- .../GeneralsOnline/NetworkMesh.cpp | 14 +- .../GeneralsOnline/OnlineServices_Init.cpp | 12 +- .../GeneralsOnline/PluginInterfaces.cpp | 2 +- 15 files changed, 277 insertions(+), 94 deletions(-) create mode 100644 Core/GameEngine/Include/GameNetwork/NetworkSettings.h diff --git a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h index 4e1693455b5..1b1d3e2731b 100644 --- a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h +++ b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h @@ -37,6 +37,9 @@ #include "GameNetwork/FrameMetrics.h" #include "GameNetwork/NetworkDefs.h" #include "GameNetwork/DisconnectManager.h" +#if defined(GENERALS_ONLINE) +#include "GameNetwork/NetworkSettings.h" +#endif class GameInfo; class NetCommandWrapperList; @@ -56,6 +59,7 @@ class ConnectionManager virtual void update(Bool isInGame); ///< Service the Connections being managed by this instance. #if defined(GENERALS_ONLINE) + const NetworkSettings &getNetworkSettings() const { return m_networkSettings; } void SeedLatencyData(int highestLatency) { for (int i = 0; i < MAX_SLOTS; ++i) { @@ -165,6 +169,10 @@ class ConnectionManager Int getPingsReceived(); private: +#if defined(GENERALS_ONLINE) + NetworkSettings m_networkSettings; +#endif + void doRelay(); void doKeepAlive(); void sendRemoteCommand(NetCommandRef *msg); diff --git a/Core/GameEngine/Include/GameNetwork/FrameMetrics.h b/Core/GameEngine/Include/GameNetwork/FrameMetrics.h index 13967e49164..fdf4edca419 100644 --- a/Core/GameEngine/Include/GameNetwork/FrameMetrics.h +++ b/Core/GameEngine/Include/GameNetwork/FrameMetrics.h @@ -59,6 +59,9 @@ class FrameMetrics { Real *m_fpsList; ///< A record of how many game logic frames per second there were for the last 60 seconds. time_t m_lastFpsTimeThing; ///< The time when the last fps entry started being recorded. Int m_fpsListIndex; ///< Index into the array of the current fps list entry being measured. +#if defined(GENERALS_ONLINE) + UnsignedInt m_fpsListLength; ///< Entries m_fpsList holds, the history length can change after construction. +#endif Real m_averageFps; ///< The current average logic fps, computed just like m_averageLatency below but with the fps numbers. // round trip time to packet router variables. diff --git a/Core/GameEngine/Include/GameNetwork/NetworkDefs.h b/Core/GameEngine/Include/GameNetwork/NetworkDefs.h index 6295493ea7d..330477fdcc5 100644 --- a/Core/GameEngine/Include/GameNetwork/NetworkDefs.h +++ b/Core/GameEngine/Include/GameNetwork/NetworkDefs.h @@ -38,7 +38,7 @@ static constexpr const Int MIN_LOGIC_FRAMES = 5; static constexpr const Int MAX_FRAMES_AHEAD = 128; #if defined(GENERALS_ONLINE) -static Int MIN_RUNAHEAD = 6; +extern Int MIN_RUNAHEAD; #else static constexpr const Int MIN_RUNAHEAD = 4; #endif diff --git a/Core/GameEngine/Include/GameNetwork/NetworkSettings.h b/Core/GameEngine/Include/GameNetwork/NetworkSettings.h new file mode 100644 index 00000000000..f09f5c1d644 --- /dev/null +++ b/Core/GameEngine/Include/GameNetwork/NetworkSettings.h @@ -0,0 +1,35 @@ +#pragma once + +#if defined(GENERALS_ONLINE) + +#include + +// Per-match game networking policy, independent of online-service lifetime. +struct NetworkSettings +{ + struct Send + { + int frames_per_packet = 2; // send interval in frames + bool flush_every_frame = true; // flush GNS after each frame + int flags = -1; // GNS send flags tried after a failed reliable send, -1 = none + } send; + struct InputDelay + { + int min_frames = 4; + int decision_interval_ms = 5000; + int fps_history_samples = 30; // per-second FPS samples averaged + int slack_percent = 30; // percent added to the measured round trip, also sets the start run ahead and the rating + + struct Formula + { + int latency_history_samples = 200; // in-band ping samples averaged + } formula; + } input_delay; + + static NetworkSettings LocalDefaults() + { + return NetworkSettings(); + } +}; + +#endif diff --git a/Core/GameEngine/Source/GameNetwork/Connection.cpp b/Core/GameEngine/Source/GameNetwork/Connection.cpp index 8ba8a3e2dcd..d81ede4a504 100644 --- a/Core/GameEngine/Source/GameNetwork/Connection.cpp +++ b/Core/GameEngine/Source/GameNetwork/Connection.cpp @@ -271,7 +271,7 @@ UnsignedInt Connection::doSend() { if ((curtime - m_lastTimeSent) < m_frameGrouping) { // DEBUG_LOG(("not sending packet, time = %d, m_lastFrameSent = %d, m_frameGrouping = %d", curtime, m_lastTimeSent, m_frameGrouping)); - NetworkLog(ELogVerbosity::LOG_DEBUG, "not sending packet, time = %d, m_lastFrameSent = %d, m_frameGrouping = %d", curtime, m_lastTimeSent, m_frameGrouping); + NetworkLog(ELogVerbosity::LOG_DEBUG, "not sending packet, time = %I64d, m_lastFrameSent = %I64d, m_frameGrouping = %I64d", (Int64)curtime, (Int64)m_lastTimeSent, (Int64)m_frameGrouping); return 0; } diff --git a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp index f347db5a83f..504fa1362c3 100644 --- a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp +++ b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp @@ -59,6 +59,10 @@ #include "../NetworkMesh.h" #include "../ngmp_interfaces.h" +#if defined(GENERALS_ONLINE) +Int MIN_RUNAHEAD = 4; +#endif + static Bool hasValidTransferFileExtension(const AsciiString& filePath) { static const char* const validExtensions[] = { @@ -272,6 +276,23 @@ ConnectionManager::ConnectionManager() */ void ConnectionManager::init() { +#if defined(GENERALS_ONLINE) + m_networkSettings = NetworkSettings::LocalDefaults(); + NGMP_OnlineServicesManager *services = NGMP_OnlineServicesManager::GetInstance(); + if (TheLAN == nullptr && TheNGMPGame != nullptr && services != nullptr) { + const ServiceConfig &overrides = services->GetServiceConfig(); + m_networkSettings.send = overrides.send; + m_networkSettings.input_delay = overrides.input_delay; + } + const NetworkSettings::InputDelay &runAhead = m_networkSettings.input_delay; + TheWritableGlobalData->m_networkFPSHistoryLength = (std::max)(1, runAhead.fps_history_samples); + TheWritableGlobalData->m_networkLatencyHistoryLength = (std::max)(1, runAhead.formula.latency_history_samples); + TheWritableGlobalData->m_networkRunAheadSlack = runAhead.slack_percent; + TheWritableGlobalData->m_networkRunAheadMetricsTime = runAhead.decision_interval_ms; + MIN_RUNAHEAD = runAhead.min_frames; + FRAME_GROUPING_CAP = (int)(m_networkSettings.send.frames_per_packet * (float)(1000.f / GENERALS_ONLINE_HIGH_FPS_LIMIT)); +#endif + // if (m_transport == nullptr) { // m_transport = new Transport; // } @@ -389,10 +410,10 @@ void ConnectionManager::reset() #endif m_packetRouterSlot = -1; - for (i = 0; i < TheGlobalData->m_networkFPSHistoryLength; ++i) { + for (i = 0; i < (UnsignedInt)MAX_SLOTS; ++i) { m_fpsAverages[i] = -1; } - for (i = 0; i < TheGlobalData->m_networkLatencyHistoryLength; ++i) { + for (i = 0; i < (UnsignedInt)MAX_SLOTS; ++i) { m_latencyAverages[i] = 0.0; } @@ -1995,7 +2016,7 @@ void ConnectionManager::setFrameGrouping(time_t frameGrouping) { // This is probably ok since the packet router should have the fastest connection of all // the players in the game. - NetworkLog(ELogVerbosity::LOG_DEBUG, "ConnectionManager::setFrameGrouping Frame grouping is %d, is packet router %d", frameGrouping, m_localSlot == m_packetRouterSlot); + NetworkLog(ELogVerbosity::LOG_DEBUG, "ConnectionManager::setFrameGrouping Frame grouping is %I64d, is packet router %d", (Int64)frameGrouping, m_localSlot == m_packetRouterSlot); if (m_localSlot == m_packetRouterSlot) { frameGrouping = frameGrouping / 2; @@ -2785,7 +2806,7 @@ void ConnectionManager::sendFrameDataToPlayer(UnsignedInt playerID, UnsignedInt } void ConnectionManager::sendSingleFrameToPlayer(UnsignedInt playerID, UnsignedInt frame) { - if ((TheGameLogic->getFrame() - FRAMES_TO_KEEP) > frame) { + if (frame + FRAMES_TO_KEEP < TheGameLogic->getFrame()) { DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("ConnectionManager::sendSingleFrameToPlayer - player %d requested frame %d when we are on frame %d, this is too far in the past.", playerID, frame, TheGameLogic->getFrame())); return; } diff --git a/Core/GameEngine/Source/GameNetwork/DisconnectManager.cpp b/Core/GameEngine/Source/GameNetwork/DisconnectManager.cpp index 9d2f1b58574..a59a6fcbc7c 100644 --- a/Core/GameEngine/Source/GameNetwork/DisconnectManager.cpp +++ b/Core/GameEngine/Source/GameNetwork/DisconnectManager.cpp @@ -214,7 +214,7 @@ void DisconnectManager::updateDisconnectStatus(ConnectionManager *conMgr) { m_haveNotifiedOtherPlayersOfCurrentFrame = TRUE; } - DEBUG_LOG(("DisconnectManager::updateDisconnectStatus - curTime = %d, m_timeOfDisconnectScreenOn = %d, curTime - m_timeOfDisconnectScreenOn = %d", curTime, m_timeOfDisconnectScreenOn, curTime - m_timeOfDisconnectScreenOn)); + DEBUG_LOG(("DisconnectManager::updateDisconnectStatus - curTime = %I64d, m_timeOfDisconnectScreenOn = %I64d, curTime - m_timeOfDisconnectScreenOn = %I64d", (Int64)curTime, (Int64)m_timeOfDisconnectScreenOn, (Int64)(curTime - m_timeOfDisconnectScreenOn))); if (m_timeOfDisconnectScreenOn != 0) { if ((curTime - m_timeOfDisconnectScreenOn) > TheGlobalData->m_networkDisconnectScreenNotifyTime) { diff --git a/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp b/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp index 0de0dd2096c..abc805ddb71 100644 --- a/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp +++ b/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp @@ -61,14 +61,17 @@ FrameMetrics::FrameMetrics() m_fpsList = NEW Real[TheGlobalData->m_networkFPSHistoryLength]; +#if defined(GENERALS_ONLINE) + m_fpsListLength = TheGlobalData->m_networkFPSHistoryLength; +#endif } FrameMetrics::~FrameMetrics() { - delete m_fpsList; + delete[] m_fpsList; m_fpsList = NULL; #if !defined(USE_NEW_FRAMEMETRIC_LOGIC) - delete m_latencyList; + delete[] m_latencyList; m_latencyList = NULL; delete[] m_pendingLatencies; @@ -115,6 +118,15 @@ void FrameMetrics::init() { #endif m_minimumCushion = -1; +#if defined(GENERALS_ONLINE) + // the game network settings can change the history length after construction + if (m_fpsListLength != TheGlobalData->m_networkFPSHistoryLength) { + delete[] m_fpsList; + m_fpsList = NEW Real[TheGlobalData->m_networkFPSHistoryLength]; + m_fpsListLength = TheGlobalData->m_networkFPSHistoryLength; + } +#endif + UnsignedInt i = 0; for (; i < TheGlobalData->m_networkFPSHistoryLength; ++i) { #if defined(GENERALS_ONLINE_HIGH_FPS_SERVER) @@ -228,8 +240,8 @@ void FrameMetrics::processLatencyResponse(UnsignedInt frame) { m_averageLatency += kvPaAir.second; } - NetworkLog(ELogVerbosity::LOG_DEBUG, "Avg latency is: %f / %d = %f", m_averageLatency, m_mapLatenciesLookup.size(), m_averageLatency /= m_mapLatenciesLookup.size()); m_averageLatency /= m_mapLatenciesLookup.size(); + NetworkLog(ELogVerbosity::LOG_DEBUG, "Avg latency is: %f over %d samples", m_averageLatency, (int)m_mapLatenciesLookup.size()); if (timeDiff > 1000) @@ -280,4 +292,16 @@ void FrameMetrics::SeedLatencyData(int latency) { m_averageFps = GENERALS_ONLINE_HIGH_FPS_LIMIT; m_averageLatency = latency / 1000.f; + +#if defined(USE_NEW_FRAMEMETRIC_LOGIC) + // the next response recomputes the average from these maps, so they must carry the seed too + for (std::unordered_map::iterator it = m_mapLatenciesLookup.begin(); it != m_mapLatenciesLookup.end(); ++it) + { + it->second = m_averageLatency; + } + for (std::map::iterator it = m_mapLatenciesSorted.begin(); it != m_mapLatenciesSorted.end(); ++it) + { + it->second = m_averageLatency; + } +#endif } diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NextGenMP_defines.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NextGenMP_defines.h index b78109bc4c4..d7ac3365899 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NextGenMP_defines.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NextGenMP_defines.h @@ -89,11 +89,7 @@ void showNotificationBox(AsciiString nick, UnicodeString message, bool bPlaySoun #define GENERALS_ONLINE_HIGH_FPS_RENDER 1 // This is optional on 30fps, but will boost/unlock the framerate, similar to gentool #endif -#if defined(GENERALS_ONLINE_HIGH_FPS_SERVER) -static int FRAME_GROUPING_CAP = 32; -#else -static int FRAME_GROUPING_CAP = 64; -#endif +extern int FRAME_GROUPING_CAP; //#define GENERALS_ONLINE_ENABLE_CONTROVERSIAL_NON_RETAIL_CHANGES 1 diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index 2096e7b3623..6b4169bb152 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -1,7 +1,10 @@ #pragma once +#include "GameNetwork/NetworkSettings.h" #include "NGMP_include.h" +#include +#include #include #include #include @@ -310,53 +313,168 @@ struct RegionResponse struct ServiceConfig { - bool retry_signalling = false; - bool use_mapped_port = true; - int min_run_ahead_frames = 4; - int ra_update_frequency_frames = 10; - bool relay_all_traffic = false; - int ra_slack_percent = 20; - int frame_grouping_frames = 2; - bool enable_host_migration = true; - - bool network_do_immediate_flush_per_frame = true; - int network_send_flags = -1; - - int network_latency_logic_model = 0; - - bool use_default_config = false; - int ra_slack_override_percent_in_default = 10; - bool do_probes = true; - bool do_replay_upload = true; - - int network_mesh_histogram_duration = 20000; - + // Peer connections: signalling, NAT traversal, and the per-connection ping history + struct Connection + { + bool relay_only = false; + int ice_engine = 2; // ICE library: 0 = library default, 1 = native, 2 = WebRTC; the key also takes "default", "native" or "webrtc" + bool use_mapped_port = true; + bool retry_signalling = true; + bool anticheat_transport = false; // game traffic through the anticheat plugin's transport + int latency_history_ms = 20000; + } connection; + + // Not read from the service: the live values + int network_latency_logic_model = 2; bool ibra_ra_tweaks = false; float ibra_minslack_default = 0.25f; float ibra_maxslack_default = 1.f; - float ibra_minslack_greaterthan300ms = 0.35f; float ibra_maxslack_greaterthan300ms = 1.f; - float ibra_minslack_greaterthan200ms = 0.3f; float ibra_maxslack_greaterthan200ms = 1.f; + using Send = NetworkSettings::Send; + using InputDelay = NetworkSettings::InputDelay; + Send send; + InputDelay input_delay; - int screenshot_width = 557; - int screenshot_height = 333; - int screenshot_upload_bytes_per_sec = 131072; // 128 KB/s - int screenshot_slot_offset_ms = 2000; + struct Lobby + { + bool host_migration = false; + } lobby; - // ICE client used for P2P NAT traversal: 0 = library default, 1 = native, 2 = WebRTC. - int ice_implementation = 2; + struct Telemetry + { + bool screenshot_probes = true; + bool replay_upload = true; + int screenshot_width = 960; + int screenshot_height = 600; + int screenshot_upload_bytes_per_sec = 131072; // 128 KB/s + int screenshot_slot_offset_ms = 2000; + } telemetry; +}; - bool use_new_networking = false; +// Reads key into value when present; a missing key keeps the default +template +inline void ReadServiceConfigValue(const nlohmann::json& j, const char* key, T& value) +{ + if (!j.is_object()) + { + throw std::invalid_argument("Service config sections must be objects"); + } + const auto it = j.find(key); + if (it != j.end() && !it->is_null()) + { + value = it->get(); + } +} - NLOHMANN_DEFINE_TYPE_INTRUSIVE_WITH_DEFAULT(ServiceConfig, retry_signalling, use_mapped_port, min_run_ahead_frames, ra_update_frequency_frames, relay_all_traffic, - ra_slack_percent, frame_grouping_frames, enable_host_migration, network_do_immediate_flush_per_frame, network_send_flags, network_latency_logic_model, - use_default_config, ra_slack_override_percent_in_default, do_probes, do_replay_upload, network_mesh_histogram_duration, - ibra_ra_tweaks, ibra_minslack_default, ibra_maxslack_default, ibra_minslack_greaterthan300ms, ibra_maxslack_greaterthan300ms, ibra_minslack_greaterthan200ms, ibra_maxslack_greaterthan200ms, - screenshot_width, screenshot_height, screenshot_upload_bytes_per_sec, screenshot_slot_offset_ms, ice_implementation, use_new_networking) -}; +// The ICE library by name or by number +inline void ReadIceEngine(const nlohmann::json& j, int& value) +{ + if (!j.is_object()) + { + throw std::invalid_argument("Service config sections must be objects"); + } + const auto it = j.find("ice_engine"); + if (it == j.end() || it->is_null()) + { + return; + } + if (it->is_string()) + { + const std::string name = it->get(); + if (name == "default") + { + value = 0; + } + else if (name == "native") + { + value = 1; + } + else if (name == "webrtc") + { + value = 2; + } + return; + } + value = it->get(); +} + +// A key the service sets replaces the compiled default, a missing key keeps it +inline void ReadServiceConfig(const nlohmann::json& j, ServiceConfig& c) +{ + if (j.contains("connection")) + { + const nlohmann::json& s = j["connection"]; + ReadServiceConfigValue(s, "relay_only", c.connection.relay_only); + ReadIceEngine(s, c.connection.ice_engine); + ReadServiceConfigValue(s, "use_mapped_port", c.connection.use_mapped_port); + ReadServiceConfigValue(s, "retry_signalling", c.connection.retry_signalling); + ReadServiceConfigValue(s, "anticheat_transport", c.connection.anticheat_transport); + ReadServiceConfigValue(s, "latency_history_ms", c.connection.latency_history_ms); + } + if (j.contains("send")) + { + const nlohmann::json& s = j["send"]; + ReadServiceConfigValue(s, "frames_per_packet", c.send.frames_per_packet); + ReadServiceConfigValue(s, "flush_every_frame", c.send.flush_every_frame); + ReadServiceConfigValue(s, "flags", c.send.flags); + } + if (j.contains("input_delay")) + { + const nlohmann::json& s = j["input_delay"]; + ReadServiceConfigValue(s, "min_frames", c.input_delay.min_frames); + ReadServiceConfigValue(s, "decision_interval_ms", c.input_delay.decision_interval_ms); + ReadServiceConfigValue(s, "fps_history_samples", c.input_delay.fps_history_samples); + ReadServiceConfigValue(s, "slack_percent", c.input_delay.slack_percent); + if (s.contains("formula")) + { + const nlohmann::json& f = s["formula"]; + ReadServiceConfigValue(f, "latency_history_samples", c.input_delay.formula.latency_history_samples); + } + } + if (j.contains("lobby")) + { + ReadServiceConfigValue(j["lobby"], "host_migration", c.lobby.host_migration); + } + if (j.contains("telemetry")) + { + const nlohmann::json& s = j["telemetry"]; + ReadServiceConfigValue(s, "screenshot_probes", c.telemetry.screenshot_probes); + ReadServiceConfigValue(s, "replay_upload", c.telemetry.replay_upload); + ReadServiceConfigValue(s, "screenshot_width", c.telemetry.screenshot_width); + ReadServiceConfigValue(s, "screenshot_height", c.telemetry.screenshot_height); + ReadServiceConfigValue(s, "screenshot_upload_bytes_per_sec", c.telemetry.screenshot_upload_bytes_per_sec); + ReadServiceConfigValue(s, "screenshot_slot_offset_ms", c.telemetry.screenshot_slot_offset_ms); + } +} + +// Keeps values from the service inside the ranges the game can run with +inline void ClampServiceConfig(ServiceConfig& c) +{ + c.connection.latency_history_ms = std::clamp(c.connection.latency_history_ms, 1000, 120000); + + c.input_delay.min_frames = std::clamp(c.input_delay.min_frames, 1, 64); + c.input_delay.slack_percent = std::clamp(c.input_delay.slack_percent, 0, 200); + c.input_delay.decision_interval_ms = std::clamp(c.input_delay.decision_interval_ms, 100, 60000); + c.input_delay.fps_history_samples = std::clamp(c.input_delay.fps_history_samples, 1, 128); + c.input_delay.formula.latency_history_samples = std::clamp(c.input_delay.formula.latency_history_samples, 128, 3600); // FrameMetrics drops pending acks when the window is smaller than the frames in flight + c.send.frames_per_packet = std::clamp(c.send.frames_per_packet, 1, 16); + + c.telemetry.screenshot_width = std::clamp(c.telemetry.screenshot_width, 16, 4096); + c.telemetry.screenshot_height = std::clamp(c.telemetry.screenshot_height, 16, 4096); +} + +inline void from_json(const nlohmann::json& j, ServiceConfig& c) +{ + if (!j.is_object()) + { + throw std::invalid_argument("Service config must be an object"); + } + ReadServiceConfig(j, c); + ClampServiceConfig(c); +} class NGMP_OnlineServicesManager { diff --git a/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp b/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp index d8c0b2c2eba..739367b2894 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp @@ -2510,7 +2510,7 @@ void WOLGameSetupMenuUpdate( WindowLayout * layout, void *userData) if (pLobbyInterface != nullptr) { ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - bool bHostMigrationEnabledOnService = serviceConf.enable_host_migration; + bool bHostMigrationEnabledOnService = serviceConf.lobby.host_migration; if (bHostMigrationEnabledOnService) { diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp index 62f7b703cba..490bfcf0e33 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp @@ -13,6 +13,12 @@ #include "GameClient/View.h" #include "../NextGenMP_defines.h" +#if defined(GENERALS_ONLINE_HIGH_FPS_SERVER) +int FRAME_GROUPING_CAP = 32; +#else +int FRAME_GROUPING_CAP = 64; +#endif + NGMPGameSlot::NGMPGameSlot() { GameSlot(); @@ -411,34 +417,6 @@ void NGMPGame::launchGame(void) //TheWritableGlobalData->m_networkPlayerTimeoutTime = 60000; //TheWritableGlobalData->m_networkDisconnectScreenNotifyTime = 2500; - // process service config - NGMP_OnlineServicesManager* pOnlineServicesMgr = NGMP_OnlineServicesManager::GetInstance(); - if (pOnlineServicesMgr != nullptr) - { - ServiceConfig& serviceConf = pOnlineServicesMgr->GetServiceConfig(); - - if (serviceConf.use_default_config) - { - TheWritableGlobalData->m_networkFPSHistoryLength = 30; - TheWritableGlobalData->m_networkLatencyHistoryLength = 200; - TheWritableGlobalData->m_networkRunAheadSlack = serviceConf.ra_slack_override_percent_in_default; // normally 10 - TheWritableGlobalData->m_networkRunAheadMetricsTime = 5000; - } - else - { - TheWritableGlobalData->m_networkFPSHistoryLength = 10; - TheWritableGlobalData->m_networkLatencyHistoryLength = 10; - - MIN_RUNAHEAD = serviceConf.min_run_ahead_frames; - - TheWritableGlobalData->m_networkRunAheadSlack = serviceConf.ra_slack_percent; - TheWritableGlobalData->m_networkRunAheadMetricsTime = serviceConf.ra_update_frequency_frames * (float)(1000.f / GENERALS_ONLINE_HIGH_FPS_LIMIT); - - FRAME_GROUPING_CAP = serviceConf.frame_grouping_frames * (float)(1000.f / GENERALS_ONLINE_HIGH_FPS_LIMIT); - } - } - - #if defined(GENERALS_ONLINE_HIGH_FPS_RENDER) TheWritableGlobalData->m_horizontalScrollSpeedFactor = NGMP_OnlineServicesManager::Settings.Camera_MoveSpeedRatio(); TheWritableGlobalData->m_verticalScrollSpeedFactor = NGMP_OnlineServicesManager::Settings.Camera_MoveSpeedRatio(); diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp index 5957924e4c2..94cbb787e1e 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp @@ -114,7 +114,7 @@ void OnSteamNetConnectionStatusChanged(SteamNetConnectionStatusChangedCallback_t const bool bPeerLeft = pJoinOrderLobby != nullptr && pJoinOrderLobby->IsJoinOrderKnown() && !pJoinOrderLobby->IsLobbyMember(userID); // a match can't leave its lobby, so keep repairing the link until the service drops the player const bool bInMatch = TheGameLogic != nullptr && TheGameLogic->isInInternetGame(); - bool bShouldRetry = serviceConf.retry_signalling && ((bInMatch && !bPeerLeft) || (!bWeJoinedLater && !bPeerLeft) || signallingAttemptsBeforeDisconnect < numSignallingAttempts); + bool bShouldRetry = serviceConf.connection.retry_signalling && ((bInMatch && !bPeerLeft) || (!bWeJoinedLater && !bPeerLeft) || signallingAttemptsBeforeDisconnect < numSignallingAttempts); bool bWasError = pInfo->m_info.m_eState == k_ESteamNetworkingConnectionState_ProblemDetectedLocally || pInfo->m_info.m_eEndReason != k_ESteamNetConnectionEnd_App_Generic; plrConnection.SetDisconnected(bWasError, pMesh, bShouldRetry && bWasError); @@ -716,12 +716,12 @@ NetworkMesh::NetworkMesh() m_strTurnTokenString = std::format("{},{}", m_strTurnToken.c_str(), m_strTurnToken.c_str()); ServiceConfig& serviceConf = pOnlineServicesMgr->GetServiceConfig(); - m_iceEnable = (g_bForceRelay || serviceConf.relay_all_traffic) + m_iceEnable = (g_bForceRelay || serviceConf.connection.relay_only) ? k_nSteamNetworkingConfig_P2P_Transport_ICE_Enable_Relay : k_nSteamNetworkingConfig_P2P_Transport_ICE_Enable_All; // 0 = library default, 1 = native, 2 = WebRTC - m_iceImplementation = (serviceConf.ice_implementation >= 0 && serviceConf.ice_implementation <= 2) ? serviceConf.ice_implementation : 2; + m_iceImplementation = (serviceConf.connection.ice_engine >= 0 && serviceConf.connection.ice_engine <= 2) ? serviceConf.connection.ice_engine : 2; NetworkLog(ELogVerbosity::LOG_RELEASE, "NetworkMesh: using ICE implementation %d (0=default, 1=native, 2=WebRTC)", m_iceImplementation); m_pSignaling = new CSignalingClient(SteamNetworkingSockets()); @@ -772,7 +772,7 @@ NetworkMesh::NetworkMesh() void NetworkMesh::Flush() { ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - bool bDoImmediateFlushPerFrame = serviceConf.network_do_immediate_flush_per_frame; + bool bDoImmediateFlushPerFrame = serviceConf.send.flush_every_frame; if (bDoImmediateFlushPerFrame) { @@ -952,7 +952,7 @@ void NetworkMesh::StartConnectionSignalling(const char* szMiddlewareID, int64_t int g_nLocalPort = 0; - int g_nVirtualPortRemote = serviceConf.use_mapped_port ? preferredPort : 0; + int g_nVirtualPortRemote = serviceConf.connection.use_mapped_port ? preferredPort : 0; // Our remote and local port don't match, so we need to set it explicitly if (g_nVirtualPortRemote != g_nLocalPort) @@ -1336,7 +1336,7 @@ int PlayerConnection::SendGamePacket(void* pBuffer, uint32_t totalDataSize) int sendFlags = k_nSteamNetworkingSend_Reliable | k_nSteamNetworkingSend_AutoRestartBrokenSession; // default from last patch ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - int netSendFlags = serviceConf.network_send_flags; + int netSendFlags = serviceConf.send.flags; NetworkLog(ELogVerbosity::LOG_DEBUG, "[GAME PACKET] Sending msg of size %ld to user %lld\n", totalDataSize, m_userID); EResult r = SteamNetworkingSockets()->SendMessageToConnection( @@ -1424,7 +1424,7 @@ void PlayerConnection::UpdateLatencyHistogram() if (NGMP_OnlineServicesManager::GetInstance() != nullptr) { ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - histogram_duration = serviceConf.network_mesh_histogram_duration; + histogram_duration = serviceConf.connection.latency_history_ms; } diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp index b2fa378fdb1..46a35dfed6f 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp @@ -122,7 +122,7 @@ static std::chrono::steady_clock::time_point GetScreenshotUploadTime() } } - const int slotOffsetMS = std::clamp(NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig().screenshot_slot_offset_ms, 0, 10000); + const int slotOffsetMS = std::clamp(NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig().telemetry.screenshot_slot_offset_ms, 0, 10000); static thread_local std::mt19937 rng{ std::random_device{}() }; const int jitterMS = std::uniform_int_distribution(0, 500)(rng); @@ -137,7 +137,7 @@ void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType scree { ServiceConfig& serviceConf = pOnlineServicesMgr->GetServiceConfig(); - if (serviceConf.do_probes) + if (serviceConf.telemetry.screenshot_probes) { CHECK_MAIN_THREAD; @@ -275,7 +275,7 @@ void NGMP_OnlineServicesManager::CommitReplay(AsciiString absoluteReplayPath) { ServiceConfig& serviceConf = pOnlineServicesMgr->GetServiceConfig(); - if (serviceConf.do_replay_upload) + if (serviceConf.telemetry.replay_upload) { NGMP_OnlineServices_LobbyInterface* pLobbyInterface = NGMP_OnlineServicesManager::GetInterface(); const uint64_t matchID = pLobbyInterface == nullptr ? 0 : pLobbyInterface->GetCurrentMatchID(); @@ -674,8 +674,8 @@ void NGMP_OnlineServicesManager::CaptureScreenshot(bool bResizeForTransmit, std: if (bResizeForTransmit) { ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - finalWidth = serviceConf.screenshot_width; - finalHeight = serviceConf.screenshot_height; + finalWidth = serviceConf.telemetry.screenshot_width; + finalHeight = serviceConf.telemetry.screenshot_height; resized.resize(finalWidth * finalHeight * channels); stbir_resize_uint8(pixels, width, height, 0, @@ -953,7 +953,7 @@ void NGMP_OnlineServicesManager::Tick() std::scoped_lock ssLock(m_ScreenshotMutex); - const int uploadBytesPerSec = std::clamp(m_ServiceConfig.screenshot_upload_bytes_per_sec, 16384, 10485760); + const int uploadBytesPerSec = std::clamp(m_ServiceConfig.telemetry.screenshot_upload_bytes_per_sec, 16384, 10485760); const std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now(); const bool bMatchOver = TheGameLogic == nullptr || !TheGameLogic->isInMultiplayerGame(); // uploads no longer compete with game traffic diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/PluginInterfaces.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/PluginInterfaces.cpp index 7d60712e8cc..c62f5f7fcf9 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/PluginInterfaces.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/PluginInterfaces.cpp @@ -507,7 +507,7 @@ bool AnticheatPlugInterface::DoesACPluginProvideSecureGameTransport() { #if defined(AC_ENABLED) ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - const bool bEnabledByConfig = serviceConf.use_new_networking; + const bool bEnabledByConfig = serviceConf.connection.anticheat_transport; // The plugin makes this same decision internally in places the game cannot // reach - notably inside the EOS anti-cheat OnMessageToPeer callback, which From 11f1f99b2d66b9ef9804437330b1c887240693b5 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Thu, 22 Oct 2026 06:59:31 +0200 Subject: [PATCH 04/12] feat(network): Measure connection latency, jitter and loss with packet feedback --- Core/GameEngine/CMakeLists.txt | 2 + .../Include/GameNetwork/Connection.h | 16 +- .../Include/GameNetwork/ConnectionManager.h | 12 + .../Include/GameNetwork/ConnectionStats.h | 162 +++++++++ .../Include/GameNetwork/NetCommandRef.h | 14 + .../Include/GameNetwork/NetworkDefs.h | 8 + .../Include/GameNetwork/NetworkInterface.h | 1 + .../Include/GameNetwork/NetworkSettings.h | 1 - .../Source/GameNetwork/Connection.cpp | 50 +-- .../Source/GameNetwork/ConnectionManager.cpp | 39 +++ .../Source/GameNetwork/ConnectionStats.cpp | 316 ++++++++++++++++++ .../Source/GameNetwork/NetCommandRef.cpp | 1 + .../GameEngine/Source/GameNetwork/Network.cpp | 14 + .../W3DDevice/GameClient/W3DDisplay.cpp | 14 + GeneralsMD/Code/GameEngine/CMakeLists.txt | 3 + .../GameNetwork/GeneralsOnline/NGMP_include.h | 3 +- .../GameNetwork/GeneralsOnline/NetBytes.h | 16 + .../GameNetwork/GeneralsOnline/NetworkMesh.h | 35 +- .../GeneralsOnline/OnlineServices_Init.h | 11 +- .../GeneralsOnline/PacketFeedback.h | 70 ++++ .../GeneralsOnline/NetworkMesh.cpp | 287 +++++++++++++--- .../GeneralsOnline/NextGenTransport.cpp | 42 ++- .../GeneralsOnline/PacketFeedback.cpp | 179 ++++++++++ .../W3DDevice/GameClient/W3DDisplay.cpp | 14 + 24 files changed, 1202 insertions(+), 108 deletions(-) create mode 100644 Core/GameEngine/Include/GameNetwork/ConnectionStats.h create mode 100644 Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp create mode 100644 GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetBytes.h create mode 100644 GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/PacketFeedback.h create mode 100644 GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/PacketFeedback.cpp diff --git a/Core/GameEngine/CMakeLists.txt b/Core/GameEngine/CMakeLists.txt index e6aee6da85f..0b0cc58d095 100644 --- a/Core/GameEngine/CMakeLists.txt +++ b/Core/GameEngine/CMakeLists.txt @@ -505,6 +505,7 @@ set(GAMEENGINE_SRC # Include/GameLogic/WeaponStatus.h Include/GameNetwork/Connection.h Include/GameNetwork/ConnectionManager.h + Include/GameNetwork/ConnectionStats.h Include/GameNetwork/DisconnectManager.h Include/GameNetwork/DownloadManager.h Include/GameNetwork/FileTransfer.h @@ -1099,6 +1100,7 @@ set(GAMEENGINE_SRC Source/GameLogic/System/RankInfo.cpp Source/GameNetwork/Connection.cpp Source/GameNetwork/ConnectionManager.cpp + Source/GameNetwork/ConnectionStats.cpp Source/GameNetwork/DisconnectManager.cpp Source/GameNetwork/DownloadManager.cpp Source/GameNetwork/FileTransfer.cpp diff --git a/Core/GameEngine/Include/GameNetwork/Connection.h b/Core/GameEngine/Include/GameNetwork/Connection.h index 9a26a5fee63..0c77332ab1b 100644 --- a/Core/GameEngine/Include/GameNetwork/Connection.h +++ b/Core/GameEngine/Include/GameNetwork/Connection.h @@ -39,12 +39,7 @@ #include "GameNetwork/User.h" #include "GameNetwork/Transport.h" #include "GameNetwork/NetPacket.h" - -#if defined(GENERALS_ONLINE) -#define CONNECTION_LATENCY_HISTORY_LENGTH 10 -#else -#define CONNECTION_LATENCY_HISTORY_LENGTH 200 -#endif +#include "GameNetwork/ConnectionStats.h" class Connection : public MemoryPoolObject { @@ -80,13 +75,13 @@ class Connection : public MemoryPoolObject void setQuitting(); Bool isQuitting() { return m_isQuitting; } + const ConnectionStats& getStats() const { return m_stats; } + #if defined(RTS_DEBUG) void debugPrintCommands(); #endif protected: - void doRetryMetrics(); - Bool m_isQuitting; UnsignedInt m_quitTime; @@ -95,11 +90,8 @@ class Connection : public MemoryPoolObject NetCommandList *m_netCommandList; time_t m_retryTime; ///< The time between sending retry packets for this connection. Time is in milliseconds. - Real m_averageLatency; ///< The average time between sending a command and receiving an ACK. - Real m_latencies[CONNECTION_LATENCY_HISTORY_LENGTH]; ///< List of the last 100 latencies. + ConnectionStats m_stats; ///< Latency, jitter and loss of this connection. time_t m_frameGrouping; ///< The minimum time between packet sends. time_t m_lastTimeSent; ///< The time of the last packet send. - Int m_numRetries; ///< The number of retries for the last second. - time_t m_retryMetricsTime; ///< The start time of the current retry metrics thing. }; diff --git a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h index 1b1d3e2731b..d6df0d5b273 100644 --- a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h +++ b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h @@ -158,6 +158,10 @@ class ConnectionManager Int getAverageFPS(); Int getSlotAverageFPS(Int slot); +#if defined(GENERALS_ONLINE) + void noteStalling(Bool stalling, UnsignedInt frame); ///< Called every network update with whether the game is waiting for commands. +#endif + const ConnectionStats* getConnectionStats(Int slot) const; ///< Returns nullptr when there is no connection for the slot. #if defined(RTS_DEBUG) void debugPrintConnectionCommands(); @@ -234,6 +238,14 @@ class ConnectionManager UnsignedInt m_smallestPacketArrivalCushion; Bool m_didSelfSlug; +#if defined(GENERALS_ONLINE) + ArrivalCushionStats m_cushionStats[MAX_SLOTS]; + StallStats m_stallStats; + Bool m_stallActive; + time_t m_stallStartTime; + UnsignedInt m_stallMissingMask; +#endif + // ----------------------------------------------------------------------------- FileCommandMap s_fileCommandMap; FileMaskMap s_fileRecipientMaskMap; diff --git a/Core/GameEngine/Include/GameNetwork/ConnectionStats.h b/Core/GameEngine/Include/GameNetwork/ConnectionStats.h new file mode 100644 index 00000000000..ac93d274ea6 --- /dev/null +++ b/Core/GameEngine/Include/GameNetwork/ConnectionStats.h @@ -0,0 +1,162 @@ +/* +** Command & Conquer Generals Zero Hour(tm) +** Copyright 2025 Electronic Arts Inc. +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +#pragma once + +#include + +// Fixed size ring of items with a time member, oldest first, that drops the items older than a window. +template +class TimedRing +{ +public: + TimedRing() { clear(); } + + void clear() { m_head = 0; m_count = 0; } + Int size() const { return m_count; } + const T& operator[](Int i) const { return m_items[(m_head + i) % N]; } + + void push(const T& item) + { + if (m_count == N) + { + m_head = (m_head + 1) % N; + --m_count; + } + m_items[(m_head + m_count) % N] = item; + ++m_count; + } + + void prune(time_t now, time_t windowMs) + { + while (m_count > 0 && now - m_items[m_head].time > windowMs) + { + m_head = (m_head + 1) % N; + --m_count; + } + } + +private: + T m_items[N]; + Int m_head; + Int m_count; +}; + +// Tracks latency, jitter and loss of one connection over a sliding time window. +class ConnectionStats +{ +public: + ConnectionStats(); + + void reset(); + void noteAcked(Real rttMs, time_t now); ///< A command that was acked on its first send. + void noteRetry(time_t now); ///< A command that was not acked in time and had to be sent again. + void noteLateAck(time_t now); ///< A retried command was acked by its earlier send after all, the retry was not a loss. + void update(time_t now); ///< Drop the samples that have left the window. + + Real getLatency() const { return m_srtt; } + Real getMinLatency() const; + Real getLatencyPercentile(Int percent, time_t since = 0) const; ///< Round trip that this share of the samples in the window, from since on, stays within + Real getJitter() const { return m_jitter; } + Real getLossPercent() const; ///< Share of retries the earlier send was not acked for + Real getLateAckPercent() const; ///< Share of retries whose earlier send was acked after the retry + Real getRetryPercent() const; ///< Share of retries, lost and late acked together + Real getLatePercent() const; ///< Share of acks that took more than twice the smoothed round trip time. + Int getSampleCount() const { return m_samples.size(); } + +private: + enum + { + MAX_SAMPLES = 1024, ///< Covers the window at 60 acked frames per second. + MAX_RETRIES = 1024 ///< As many as acks, under heavy loss a smaller ring undercounts the retries of the window. + }; + + struct Sample + { + time_t time; + Real rtt; + Bool late; + }; + + struct Stamp + { + time_t time; + }; + + TimedRing m_samples; + TimedRing m_retries; + TimedRing m_lateAcks; + + Real m_srtt; + Real m_jitter; + Real m_previousRtt; + Bool m_hasRtt; +}; + +// Tracks how many frames before its execution frame the frame commands of one sender arrive, +// over the same sliding window as ConnectionStats. Zero or less means the frame had to wait. +class ArrivalCushionStats +{ +public: + ArrivalCushionStats(); + + void reset(); + void note(Int cushion, time_t now); + void update(time_t now); + + Int getSampleCount() const { return m_samples.size(); } + Int getPercentile(Int percent) const; ///< Cushion that percent of the samples are at or above the lowest value of. + +private: + enum { MAX_SAMPLES = 1024 }; + + struct Sample + { + time_t time; + Int cushion; + }; + + TimedRing m_samples; +}; + +// Frames the game wanted to advance but had to wait for commands, over the same sliding window. +class StallStats +{ +public: + StallStats(); + + void reset(); + void note(Int durationMs, UnsignedInt missingSlotMask, time_t now); ///< One finished stall and the slots whose commands were missing when it began. + void update(time_t now); + + Int getCount() const { return m_events.size(); } + Int getTotalMs() const; + Int getCountForSlot(Int slot) const; + +private: + enum { MAX_EVENTS = 512 }; + + struct Event + { + time_t time; + Int ms; + UnsignedInt slotMask; + }; + + TimedRing m_events; +}; diff --git a/Core/GameEngine/Include/GameNetwork/NetCommandRef.h b/Core/GameEngine/Include/GameNetwork/NetCommandRef.h index 4a3ae4226ca..86603e10f3e 100644 --- a/Core/GameEngine/Include/GameNetwork/NetCommandRef.h +++ b/Core/GameEngine/Include/GameNetwork/NetCommandRef.h @@ -62,12 +62,16 @@ class NetCommandRef : public MemoryPoolObject time_t getTimeLastSent() const; void setTimeLastSent(time_t timeLastSent); + Bool isRetried() const; + void setRetried(Bool retried); + protected: NetCommandMsg *m_msg; NetCommandRef *m_next; NetCommandRef *m_prev; UnsignedByte m_relay; ///< Need this in the command reference since the relay value will be different depending on where this particular reference is being sent. time_t m_timeLastSent; + Bool m_retried; ///< Set once this command was sent again, so its last send time is not its first. #ifdef DEBUG_NETCOMMANDREF UnsignedInt m_id; @@ -138,6 +142,16 @@ inline void NetCommandRef::setTimeLastSent(time_t timeLastSent) m_timeLastSent = timeLastSent; } +inline Bool NetCommandRef::isRetried() const +{ + return m_retried; +} + +inline void NetCommandRef::setRetried(Bool retried) +{ + m_retried = retried; +} + /** * Set the send relay for this reference of the command. */ diff --git a/Core/GameEngine/Include/GameNetwork/NetworkDefs.h b/Core/GameEngine/Include/GameNetwork/NetworkDefs.h index 330477fdcc5..af159f71874 100644 --- a/Core/GameEngine/Include/GameNetwork/NetworkDefs.h +++ b/Core/GameEngine/Include/GameNetwork/NetworkDefs.h @@ -56,6 +56,9 @@ enum ConnectionNumbers CPP_11(: Int) static constexpr const Int MAX_SLOTS = MAX_PLAYER + 1; +// Sliding window over which connection latency, jitter, loss and cushion are evaluated. +static constexpr const Int NETWORK_STATS_WINDOW_MS = 10000; + #pragma pack(push, 1) struct TransportMessageHeader { @@ -207,7 +210,12 @@ enum PlayerLeaveCode CPP_11(: Int) { }; // Magic number for identifying a Generals packet. +#if defined(GENERALS_ONLINE) +// 0xF00E: packets carry the sequence envelope; builds without it fail on the magic number +static constexpr const UnsignedShort GENERALS_MAGIC_NUMBER = 0xF00E; +#else static constexpr const UnsignedShort GENERALS_MAGIC_NUMBER = 0xF00D; +#endif // The number of fps history entries. //static constexpr const Int NETWORK_FPS_HISTORY_LENGTH = 30; diff --git a/Core/GameEngine/Include/GameNetwork/NetworkInterface.h b/Core/GameEngine/Include/GameNetwork/NetworkInterface.h index 606429450dd..c8ff8f0122f 100644 --- a/Core/GameEngine/Include/GameNetwork/NetworkInterface.h +++ b/Core/GameEngine/Include/GameNetwork/NetworkInterface.h @@ -100,6 +100,7 @@ class NetworkInterface : public SubsystemInterface virtual Int getAverageFPS() = 0; virtual Int getSlotAverageFPS(Int slot) = 0; + virtual const ConnectionStats* getConnectionStats(Int slot) const = 0; ///< Latency, jitter and loss of the connection to a slot, nullptr when there is none. #if defined(GENERALS_ONLINE) virtual void SeedLatencyData(int highestLatency) = 0; diff --git a/Core/GameEngine/Include/GameNetwork/NetworkSettings.h b/Core/GameEngine/Include/GameNetwork/NetworkSettings.h index f09f5c1d644..af102d9109d 100644 --- a/Core/GameEngine/Include/GameNetwork/NetworkSettings.h +++ b/Core/GameEngine/Include/GameNetwork/NetworkSettings.h @@ -11,7 +11,6 @@ struct NetworkSettings { int frames_per_packet = 2; // send interval in frames bool flush_every_frame = true; // flush GNS after each frame - int flags = -1; // GNS send flags tried after a failed reliable send, -1 = none } send; struct InputDelay { diff --git a/Core/GameEngine/Source/GameNetwork/Connection.cpp b/Core/GameEngine/Source/GameNetwork/Connection.cpp index d81ede4a504..2831f12c436 100644 --- a/Core/GameEngine/Source/GameNetwork/Connection.cpp +++ b/Core/GameEngine/Source/GameNetwork/Connection.cpp @@ -48,12 +48,6 @@ Connection::Connection() { m_frameGrouping = 1; m_isQuitting = false; m_quitTime = 0; - m_averageLatency = 0.0f; - Int i; - for(i = 0; i < CONNECTION_LATENCY_HISTORY_LENGTH; i++) - { - m_latencies[i] = 0.0f; - } } /** @@ -84,13 +78,7 @@ void Connection::init() { m_lastTimeSent = 0; m_frameGrouping = 1; - m_numRetries = 0; - m_retryMetricsTime = 0; - - for (Int i = 0; i < CONNECTION_LATENCY_HISTORY_LENGTH; ++i) { - m_latencies[i] = 0; - } - m_averageLatency = 0; + m_stats.reset(); m_isQuitting = FALSE; m_quitTime = 0; } @@ -261,6 +249,8 @@ UnsignedInt Connection::doSend() { time_t curtime = timeGetTime(); Bool couldQueue = TRUE; + m_stats.update(curtime); + // Do this check first, since it's an important fail-safe if (m_isQuitting && curtime > m_quitTime + MaxQuitFlushTime) { @@ -297,9 +287,9 @@ UnsignedInt Connection::doSend() { // the msg command was added to the packet. if (CommandRequiresAck(msg->getCommand())) { if (timeLastSent != -1) { - ++m_numRetries; + m_stats.noteRetry(curtime); + msg->setRetried(TRUE); } - doRetryMetrics(); msg->setTimeLastSent(curtime); } else { m_netCommandList->removeMessage(msg); @@ -376,13 +366,16 @@ NetCommandRef * Connection::processAck(UnsignedShort commandID, UnsignedByte ori } #endif - Int index = temp->getCommand()->getID() % CONNECTION_LATENCY_HISTORY_LENGTH; - m_averageLatency -= ((Real)(m_latencies[index])) / CONNECTION_LATENCY_HISTORY_LENGTH; - Real lat = timeGetTime() - temp->getTimeLastSent(); - - NetworkLog(ELogVerbosity::LOG_DEBUG, "Latency calc is %f, avg is %f", lat, m_averageLatency); - m_averageLatency += lat / CONNECTION_LATENCY_HISTORY_LENGTH; - m_latencies[index] = lat; + // A retried command was last sent after its first send, so its round trip time would be too short. + const time_t now = timeGetTime(); + if (temp->isRetried()) { + // an ack sooner after the retry than any round trip so far answers the earlier send: the retry went out for a late ack, not a loss + if ((m_stats.getSampleCount() > 0) && ((now - temp->getTimeLastSent()) < (time_t)(0.9f * m_stats.getMinLatency()))) { + m_stats.noteLateAck(now); + } + } else { + m_stats.noteAcked((Real)(now - temp->getTimeLastSent()), now); + } #if defined(RTS_DEBUG) if (doDebug == TRUE) { @@ -398,19 +391,6 @@ void Connection::setFrameGrouping(time_t frameGrouping) { // m_retryTime = frameGrouping * 4; } -void Connection::doRetryMetrics() { - static Int numSeconds = 0; - time_t curTime = timeGetTime(); - - if ((curTime - m_retryMetricsTime) > 10000) { - m_retryMetricsTime = curTime; - ++numSeconds; -// DEBUG_LOG(("Retries in the last 10 seconds = %d, average latency = %fms", m_numRetries, m_averageLatency)); - m_numRetries = 0; -// m_retryTime = m_averageLatency * 1.5; - } -} - #if defined(RTS_DEBUG) void Connection::debugPrintCommands() { NetCommandRef *ref = m_netCommandList->getFirstMessage(); diff --git a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp index 504fa1362c3..d569542efdb 100644 --- a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp +++ b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp @@ -339,6 +339,11 @@ void ConnectionManager::init() m_latencyAverages[i] = 0.0; // using zero since all floating point standards should be able to specify 0.0 accurately. } m_smallestPacketArrivalCushion = -1; +#if defined(GENERALS_ONLINE) + m_stallActive = FALSE; + m_stallStartTime = 0; + m_stallMissingMask = 0; +#endif m_frameMetrics.init(); @@ -1002,6 +1007,11 @@ void ConnectionManager::processFrameInfo(NetFrameCommandMsg *msg) { if (m_frameData[playerID] != nullptr) { // DEBUG_LOG(("ConnectionManager::processFrameInfo - player %d, frame %d, command count %d, received on frame %d", playerID, msg->getExecutionFrame(), msg->getCommandCount(), TheGameLogic->getFrame())); m_frameData[playerID]->setFrameCommandCount(msg->getExecutionFrame(), msg->getCommandCount()); +#if defined(GENERALS_ONLINE) + if ((playerID != (UnsignedInt)m_localSlot) && (TheGameLogic != nullptr)) { + m_cushionStats[playerID].note((Int)msg->getExecutionFrame() - (Int)TheGameLogic->getFrame(), timeGetTime()); + } +#endif } } } @@ -2742,6 +2752,13 @@ Int ConnectionManager::getSlotAverageFPS(Int slot) { return m_fpsAverages[slot]; } +const ConnectionStats* ConnectionManager::getConnectionStats(Int slot) const { + if ((slot < 0) || (slot >= MAX_SLOTS) || (m_connections[slot] == nullptr)) { + return nullptr; + } + return &m_connections[slot]->getStats(); +} + #if defined(RTS_DEBUG) void ConnectionManager::debugPrintConnectionCommands() { DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("ConnectionManager::debugPrintConnectionCommands - begin commands")); @@ -2873,3 +2890,25 @@ void ConnectionManager::requestFrameDataResend(Int playerID, UnsignedInt frame) msg->detach(); } + +#if defined(GENERALS_ONLINE) +void ConnectionManager::noteStalling(Bool stalling, UnsignedInt frame) { + time_t now = timeGetTime(); + + if (stalling && !m_stallActive) { + // list the senders whose commands for this frame are missing + m_stallMissingMask = 0; + for (Int i = 0; i < MAX_SLOTS; ++i) { + if ((m_frameData[i] != nullptr) && (m_frameData[i]->getIsQuitting() == FALSE) + && (m_frameData[i]->getFrameCommandCount(frame) != m_frameData[i]->getCommandCount(frame))) { + m_stallMissingMask |= (1u << i); + } + } + m_stallActive = TRUE; + m_stallStartTime = now; + } else if (!stalling && m_stallActive) { + m_stallActive = FALSE; + m_stallStats.note((Int)(now - m_stallStartTime), m_stallMissingMask, now); + } +} +#endif diff --git a/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp b/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp new file mode 100644 index 00000000000..c0c99c3673a --- /dev/null +++ b/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp @@ -0,0 +1,316 @@ +/* +** Command & Conquer Generals Zero Hour(tm) +** Copyright 2025 Electronic Arts Inc. +** +** This program is free software: you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation, either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program. If not, see . +*/ + +#include "PreRTS.h" // This must go first in EVERY cpp file in the GameEngine + +#include "GameNetwork/ConnectionStats.h" +#include "GameNetwork/NetworkDefs.h" + +#include + +ConnectionStats::ConnectionStats() +{ + reset(); +} + +void ConnectionStats::reset() +{ + m_samples.clear(); + m_retries.clear(); + m_lateAcks.clear(); + m_srtt = 0.0f; + m_jitter = 0.0f; + m_previousRtt = 0.0f; + m_hasRtt = FALSE; +} + +void ConnectionStats::noteAcked(Real rttMs, time_t now) +{ + if (rttMs < 0.0f) + { + rttMs = 0.0f; + } + + Bool late = FALSE; + if (m_hasRtt) + { + // RFC 3550 interarrival jitter and the TCP smoothed round trip time. + const Real delta = rttMs - m_previousRtt; + const Real deviation = delta < 0.0f ? -delta : delta; + // late = more than 2 x srtt, floored at 20 ms + const Real lateMargin = m_srtt > 20.0f ? m_srtt : 20.0f; + late = (rttMs - m_srtt) > lateMargin; + m_jitter += (deviation - m_jitter) / 16.0f; + m_srtt += (rttMs - m_srtt) / 8.0f; + } + else + { + m_srtt = rttMs; + m_hasRtt = TRUE; + } + m_previousRtt = rttMs; + + Sample sample; + sample.time = now; + sample.rtt = rttMs; + sample.late = late; + m_samples.push(sample); +} + +void ConnectionStats::noteRetry(time_t now) +{ + Stamp stamp; + stamp.time = now; + m_retries.push(stamp); +} + +void ConnectionStats::noteLateAck(time_t now) +{ + Stamp stamp; + stamp.time = now; + m_lateAcks.push(stamp); +} + +void ConnectionStats::update(time_t now) +{ + m_samples.prune(now, NETWORK_STATS_WINDOW_MS); + m_retries.prune(now, NETWORK_STATS_WINDOW_MS); + m_lateAcks.prune(now, NETWORK_STATS_WINDOW_MS); +} + +Real ConnectionStats::getMinLatency() const +{ + if (m_samples.size() == 0) + { + return 0.0f; + } + + Real result = m_samples[0].rtt; + for (Int i = 1; i < m_samples.size(); ++i) + { + if (m_samples[i].rtt < result) + { + result = m_samples[i].rtt; + } + } + return result; +} + +Real ConnectionStats::getLatencyPercentile(Int percent, time_t since) const +{ + Real sorted[MAX_SAMPLES]; + Int count = 0; + for (Int i = 0; i < m_samples.size(); ++i) + { + if (m_samples[i].time >= since) + { + sorted[count] = m_samples[i].rtt; + ++count; + } + } + if (count == 0) + { + return 0.0f; + } + std::sort(sorted, sorted + count); + + Int index = (count - 1) * percent / 100; + if (index < 0) + { + index = 0; + } + if (index >= count) + { + index = count - 1; + } + return sorted[index]; +} + +Real ConnectionStats::getRetryPercent() const +{ + const Int total = m_samples.size() + m_retries.size(); + if (total == 0) + { + return 0.0f; + } + + return 100.0f * m_retries.size() / total; +} + +Real ConnectionStats::getLateAckPercent() const +{ + const Int total = m_samples.size() + m_retries.size(); + if (total == 0) + { + return 0.0f; + } + + const Int lateAcks = m_lateAcks.size() < m_retries.size() ? m_lateAcks.size() : m_retries.size(); + return 100.0f * lateAcks / total; +} + +Real ConnectionStats::getLossPercent() const +{ + const Int total = m_samples.size() + m_retries.size(); + if (total == 0) + { + return 0.0f; + } + + const Int lost = m_retries.size() > m_lateAcks.size() ? m_retries.size() - m_lateAcks.size() : 0; + return 100.0f * lost / total; +} + +Real ConnectionStats::getLatePercent() const +{ + if (m_samples.size() == 0) + { + return 0.0f; + } + + Int lateCount = 0; + for (Int i = 0; i < m_samples.size(); ++i) + { + if (m_samples[i].late) + { + ++lateCount; + } + } + return 100.0f * lateCount / m_samples.size(); +} + +ArrivalCushionStats::ArrivalCushionStats() +{ + reset(); +} + +void ArrivalCushionStats::reset() +{ + m_samples.clear(); +} + +void ArrivalCushionStats::note(Int cushion, time_t now) +{ + update(now); + + Sample sample; + sample.time = now; + sample.cushion = cushion; + m_samples.push(sample); +} + +void ArrivalCushionStats::update(time_t now) +{ + m_samples.prune(now, NETWORK_STATS_WINDOW_MS); +} + +Int ArrivalCushionStats::getPercentile(Int percent) const +{ + if (m_samples.size() == 0) + { + return 0; + } + + // counting sort over a clamped range, the values are small frame counts + enum { LOW = -64, RANGE = 256 }; + Int counts[RANGE]; + for (Int r = 0; r < RANGE; ++r) + { + counts[r] = 0; + } + + for (Int i = 0; i < m_samples.size(); ++i) + { + Int cushion = m_samples[i].cushion; + if (cushion < LOW) + { + cushion = LOW; + } + if (cushion > LOW + RANGE - 1) + { + cushion = LOW + RANGE - 1; + } + ++counts[cushion - LOW]; + } + + Int target = (m_samples.size() * percent + 99) / 100; + if (target < 1) + { + target = 1; + } + + Int seen = 0; + for (Int q = 0; q < RANGE; ++q) + { + seen += counts[q]; + if (seen >= target) + { + return q + LOW; + } + } + return LOW + RANGE - 1; +} + +StallStats::StallStats() +{ + reset(); +} + +void StallStats::reset() +{ + m_events.clear(); +} + +void StallStats::note(Int durationMs, UnsignedInt missingSlotMask, time_t now) +{ + update(now); + + Event event; + event.time = now; + event.ms = durationMs; + event.slotMask = missingSlotMask; + m_events.push(event); +} + +void StallStats::update(time_t now) +{ + m_events.prune(now, NETWORK_STATS_WINDOW_MS); +} + +Int StallStats::getTotalMs() const +{ + Int total = 0; + for (Int i = 0; i < m_events.size(); ++i) + { + total += m_events[i].ms; + } + return total; +} + +Int StallStats::getCountForSlot(Int slot) const +{ + Int count = 0; + for (Int i = 0; i < m_events.size(); ++i) + { + if (m_events[i].slotMask & (1u << slot)) + { + ++count; + } + } + return count; +} diff --git a/Core/GameEngine/Source/GameNetwork/NetCommandRef.cpp b/Core/GameEngine/Source/GameNetwork/NetCommandRef.cpp index ba59cab5694..3c470673176 100644 --- a/Core/GameEngine/Source/GameNetwork/NetCommandRef.cpp +++ b/Core/GameEngine/Source/GameNetwork/NetCommandRef.cpp @@ -49,6 +49,7 @@ NetCommandRef::NetCommandRef(NetCommandMsg *msg) m_prev = nullptr; m_relay = 0; m_timeLastSent = -1; + m_retried = FALSE; #ifdef DEBUG_NETCOMMANDREF m_id = ++refNum; diff --git a/Core/GameEngine/Source/GameNetwork/Network.cpp b/Core/GameEngine/Source/GameNetwork/Network.cpp index a149bca7179..9dc70133a97 100644 --- a/Core/GameEngine/Source/GameNetwork/Network.cpp +++ b/Core/GameEngine/Source/GameNetwork/Network.cpp @@ -165,6 +165,7 @@ class Network : public NetworkInterface virtual Int getAverageFPS() override { return m_conMgr->getAverageFPS(); } virtual Int getSlotAverageFPS(Int slot) override; + virtual const ConnectionStats* getConnectionStats(Int slot) const override; virtual void attachTransport(Transport *transport) override; virtual void initTransport() override; @@ -791,6 +792,12 @@ void Network::update() QueryPerformanceCounter((LARGE_INTEGER*)&curTime); m_isStalling = curTime >= m_nextFrameTime; } + +#if defined(GENERALS_ONLINE) + if (m_conMgr != nullptr && m_localStatus == NETLOCALSTATUS_INGAME) { + m_conMgr->noteStalling(m_isStalling, TheGameLogic->getFrame()); + } +#endif } void Network::liteupdate() { @@ -1094,6 +1101,13 @@ Int Network::getSlotAverageFPS(Int slot) { return -1; } +const ConnectionStats* Network::getConnectionStats(Int slot) const { + if (m_conMgr != nullptr) { + return m_conMgr->getConnectionStats(slot); + } + return nullptr; +} + #if defined(RTS_DEBUG) void Network::toggleNetworkOn() { if (m_networkOn == TRUE) { diff --git a/Generals/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp b/Generals/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp index 38595af3fcf..6ef77a53e79 100644 --- a/Generals/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp +++ b/Generals/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp @@ -1446,6 +1446,20 @@ void W3DDisplay::gatherDebugStats() // Network performance stats unibuffer.format(L"Run Ahead: %d, Net FPS: %d, Packet arrival cushion: %d", TheNetwork->getRunAhead(), TheNetwork->getFrameRate(), TheNetwork->getPacketArrivalCushion()); + + // per-connection latency, jitter and loss + for (Int slot = 0; slot < MAX_SLOTS; ++slot) { + const ConnectionStats* stats = TheNetwork->getConnectionStats(slot); + if (stats != nullptr && stats->getSampleCount() > 0) { + UnicodeString statsstr; + if (stats->getLatePercent() > 0.0f) { + statsstr.format(L" P%d %.0fms j%.1f l%.0f/%.0f%%", slot, stats->getLatency(), stats->getJitter(), stats->getLossPercent(), stats->getLatePercent()); + } else { + statsstr.format(L" P%d %.0fms j%.1f l%.0f%%", slot, stats->getLatency(), stats->getJitter(), stats->getLossPercent()); + } + unibuffer.concat(statsstr); + } + } m_displayStrings[NetStats]->setText( unibuffer ); // Client frame rate averages for all players in the game. This only works right for the packet router. diff --git a/GeneralsMD/Code/GameEngine/CMakeLists.txt b/GeneralsMD/Code/GameEngine/CMakeLists.txt index a84c0935e98..5bbf0e7c005 100644 --- a/GeneralsMD/Code/GameEngine/CMakeLists.txt +++ b/GeneralsMD/Code/GameEngine/CMakeLists.txt @@ -1141,6 +1141,8 @@ set(GAMEENGINE_SRC Include/GameNetwork/GeneralsOnline/Vendor/DiscordRPC/discord_rpc.h Include/GameNetwork/GeneralsOnline/NetworkBitstream.h Include/GameNetwork/GeneralsOnline/NetworkMesh.h + Include/GameNetwork/GeneralsOnline/NetBytes.h + Include/GameNetwork/GeneralsOnline/PacketFeedback.h Include/GameNetwork/GeneralsOnline/NetworkPacket.h Include/GameNetwork/GeneralsOnline/NextGenMP_defines.h Include/GameNetwork/GeneralsOnline/NGMPGame.h @@ -1167,6 +1169,7 @@ set(GAMEENGINE_SRC Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp + Source/GameNetwork/GeneralsOnline/PacketFeedback.cpp Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp Source/GameNetwork/GeneralsOnline/OnlineServices_Auth.cpp Source/GameNetwork/GeneralsOnline/OnlineServices_LobbyInterface.cpp diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h index 342de5a1087..b22c0ced24e 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h @@ -10,7 +10,8 @@ enum class ELogVerbosity enum class ENetworkChannel : BYTE { NETWORK_CHANNEL_GAME = 0, - NETWORK_CHANNEL_AC = 1 + NETWORK_CHANNEL_AC = 1, + NETWORK_CHANNEL_PING = 2 }; #define CHECK_MAIN_THREAD assert(std::this_thread::get_id() == NGMP_OnlineServicesManager::g_MainThreadID); diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetBytes.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetBytes.h new file mode 100644 index 00000000000..c173e617a29 --- /dev/null +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetBytes.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +// Little endian wire values, the game runs on little endian targets only + +inline void WriteU16(uint8_t* out, uint16_t value) +{ + out[0] = static_cast(value & 0xFF); + out[1] = static_cast(value >> 8); +} + +inline uint16_t ReadU16(const uint8_t* in) +{ + return static_cast(in[0] | (in[1] << 8)); +} diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h index fe3c7ac9c6c..7bae0762a64 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h @@ -6,6 +6,9 @@ #include #include #include "PluginInterfaces.h" +#include "Lib/BaseType.h" +#include "GameNetwork/ConnectionStats.h" +#include "GameNetwork/GeneralsOnline/PacketFeedback.h" class NetRoom_ChatMessagePacket; @@ -50,6 +53,12 @@ class PlayerConnection void UpdateLatencyHistogram(); + void TickProbe(int64_t nowMs); + void HandleProbe(const uint8_t* payload, size_t len, int64_t arrivalMs); + void ResetProbe(); + static int64_t GetArrivalMs(const SteamNetworkingMessage_t* msg); ///< transport receive time in steady_clock ms + ConnectionStats m_probeStats; + void Close(); bool IsIPV4(); @@ -66,7 +75,7 @@ class PlayerConnection m_State != EConnectionState::CONNECTION_DISCONNECTED; } - int Recv(SteamNetworkingMessage_t** pMsg); + int Recv(SteamNetworkingMessage_t** pMsg, int maxMessages = 255); int GetHighestHistoricalLatency() { @@ -96,14 +105,18 @@ class PlayerConnection EConnectionState m_State = EConnectionState::NOT_CONNECTED; - int64_t pingSent = -1; - int m_SignallingAttempts = 0; // when signalling for this connection started, for connect-time logging int64_t m_connectStartedMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); int GetLatency(); + PacketFeedback m_feedback; + size_t ConsumeEnvelope(const uint8_t* data, size_t len); ///< envelope size, 0 if malformed + int64_t m_lastPacketMs = 0; ///< steady_clock ms of the last enveloped packet, 0 before the first + int SendEnveloped(ENetworkChannel channel, const uint8_t* payload, size_t len, int sendFlags); ///< returns an EResult + bool GetOutgoingLoss(int windowMs, PacketFeedback::Summary& summary); ///< our packets to the peer, false without feedback + bool GetIncomingLoss(int windowMs, PacketFeedback::Summary& summary); ///< the peer's packets to us int GetJitter(); float GetConnectionQuality(); int ComputeConnectionScore(); @@ -118,6 +131,22 @@ class PlayerConnection std::string m_strMiddlewareID = std::string("NOT SET"); void LiteUpdateForAC(); + +private: + static constexpr size_t k_probeRingSize = 64; + static constexpr uint16_t k_noProbeSeq = 0xFFFF; + + struct OutstandingProbe + { + uint16_t seq = k_noProbeSeq; // k_noProbeSeq marks a free slot + int64_t sendTimeMs = 0; + }; + + OutstandingProbe m_probeRing[k_probeRingSize]; // indexed by seq % k_probeRingSize + uint16_t m_nextProbeSeq = 0; + uint16_t m_lastPeerSeq = k_noProbeSeq; // last probe received from the peer + int64_t m_lastPeerSeqRecvMs = 0; + int64_t m_nextProbeMs = 0; }; struct LobbyMemberEntry; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index 6b4169bb152..9e64d15f6c4 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -313,7 +313,7 @@ struct RegionResponse struct ServiceConfig { - // Peer connections: signalling, NAT traversal, and the per-connection ping history + // Peer connections: signalling, NAT traversal, latency probes, and the per-connection ping history struct Connection { bool relay_only = false; @@ -321,6 +321,8 @@ struct ServiceConfig bool use_mapped_port = true; bool retry_signalling = true; bool anticheat_transport = false; // game traffic through the anticheat plugin's transport + int probe_lobby_interval_ms = 250; // time between latency probes sent to each peer while not in a match + int probe_match_interval_ms = 1000; // the same in a match, the command acks already measure the round trip int latency_history_ms = 20000; } connection; @@ -412,6 +414,8 @@ inline void ReadServiceConfig(const nlohmann::json& j, ServiceConfig& c) ReadServiceConfigValue(s, "use_mapped_port", c.connection.use_mapped_port); ReadServiceConfigValue(s, "retry_signalling", c.connection.retry_signalling); ReadServiceConfigValue(s, "anticheat_transport", c.connection.anticheat_transport); + ReadServiceConfigValue(s, "probe_lobby_interval_ms", c.connection.probe_lobby_interval_ms); + ReadServiceConfigValue(s, "probe_match_interval_ms", c.connection.probe_match_interval_ms); ReadServiceConfigValue(s, "latency_history_ms", c.connection.latency_history_ms); } if (j.contains("send")) @@ -419,7 +423,6 @@ inline void ReadServiceConfig(const nlohmann::json& j, ServiceConfig& c) const nlohmann::json& s = j["send"]; ReadServiceConfigValue(s, "frames_per_packet", c.send.frames_per_packet); ReadServiceConfigValue(s, "flush_every_frame", c.send.flush_every_frame); - ReadServiceConfigValue(s, "flags", c.send.flags); } if (j.contains("input_delay")) { @@ -453,7 +456,9 @@ inline void ReadServiceConfig(const nlohmann::json& j, ServiceConfig& c) // Keeps values from the service inside the ranges the game can run with inline void ClampServiceConfig(ServiceConfig& c) { - c.connection.latency_history_ms = std::clamp(c.connection.latency_history_ms, 1000, 120000); + c.connection.latency_history_ms = (std::max)(c.connection.latency_history_ms, 1000); + c.connection.probe_lobby_interval_ms = std::clamp(c.connection.probe_lobby_interval_ms, 50, 5000); + c.connection.probe_match_interval_ms = std::clamp(c.connection.probe_match_interval_ms, 50, 10000); c.input_delay.min_frames = std::clamp(c.input_delay.min_frames, 1, 64); c.input_delay.slack_percent = std::clamp(c.input_delay.slack_percent, 0, 200); diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/PacketFeedback.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/PacketFeedback.h new file mode 100644 index 00000000000..c5fc1ba138e --- /dev/null +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/PacketFeedback.h @@ -0,0 +1,70 @@ +#pragma once + +#include +#include +#include + +// Per-peer packet envelope after the channel byte: u16 sequence, u8 flags (bit 7 always set, bit 0 = feedback record follows), +// optional feedback { u16 highest seen, u8 expected, u8 received, u8 longest lost run } over the last 64 sequence numbers, +// sent at most every 250 ms. +class PacketFeedback +{ +public: + enum + { + ENVELOPE_BASE_SIZE = 3, + FEEDBACK_SIZE = 5, + ENVELOPE_MAX_SIZE = ENVELOPE_BASE_SIZE + FEEDBACK_SIZE + }; + + struct Summary + { + Summary() : samples(0), lossPercent(0.0f), longestRun(0) {} + + int samples; ///< Reports in the window, 0 when there is nothing to say. + float lossPercent; + int longestRun; ///< Longest run of consecutive lost packets in any report. + }; + + PacketFeedback(); + + void Reset(); + + size_t WriteEnvelope(uint8_t* out, int64_t nowMs); + + /// The envelope written last never left this machine, its sequence number is not a lost packet. + void CancelEnvelope(); + + /// Returns the envelope size, 0 if malformed. + size_t ReadEnvelope(const uint8_t* in, size_t len, int64_t nowMs); + + Summary GetOutgoing(int64_t nowMs, int windowMs) const; + + Summary GetIncoming(int64_t nowMs, int windowMs) const; + +private: + struct Report + { + int64_t timeMs; + uint8_t expected; + uint8_t received; + uint8_t longestRun; + }; + + static void Push(std::deque& reports, const Report& report, int64_t nowMs); + static Summary Summarize(const std::deque& reports, int64_t nowMs, int windowMs); + + enum { MAX_WINDOW_MS = 60000, FEEDBACK_INTERVAL_MS = 250 }; + + uint16_t m_nextSequence; + uint32_t m_sentCount; + + bool m_haveHighest; + uint16_t m_highest; + uint64_t m_mask; ///< Bit n is set when highest - n was received. + uint8_t m_span; ///< How many sequence numbers of the window have been seen, at most 64. + int64_t m_lastFeedbackMs; + + std::deque m_outgoing; ///< reports from the peer about our packets + std::deque m_incoming; ///< taken when we send our record +}; diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp index 94cbb787e1e..25048c38a0f 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp @@ -1,5 +1,6 @@ #include "GameNetwork/GeneralsOnline/NetworkMesh.h" #include "GameNetwork/GeneralsOnline/NGMP_include.h" +#include "GameNetwork/GeneralsOnline/NetBytes.h" #include "GameNetwork/GeneralsOnline/NGMP_interfaces.h" #include @@ -19,6 +20,11 @@ bool g_bForceRelay = false; UnsignedInt m_exeCRCOriginal = 0; +static int64_t NowSteadyMs() +{ + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); +} + // Pool for deferred deletion of ConnectionSignaling objects; avoids "delete this" races during // async Steam callbacks. static std::mutex g_pendingDeletionMutex; @@ -1171,10 +1177,13 @@ void NetworkMesh::Tick() m_pSignaling->Poll(); } + const int64_t nowMs = NowSteadyMs(); + // update connection histograms for (auto& kvPair : m_mapConnections) { PlayerConnection& conn = kvPair.second; + conn.TickProbe(nowMs); conn.UpdateLatencyHistogram(); } @@ -1248,6 +1257,12 @@ void PlayerConnection::LiteUpdateForAC() msg->Release(); continue; } + else if (netChannel == ENetworkChannel::NETWORK_CHANNEL_PING) + { + HandleProbe(vecData.data() + sizeof(ENetworkChannel), numBytes - sizeof(ENetworkChannel), GetArrivalMs(msg)); + msg->Release(); + continue; + } } else if (numBytes != -1 && numBytes < sizeof(ENetworkChannel)) { @@ -1264,6 +1279,160 @@ void PlayerConnection::LiteUpdateForAC() } } +// Probe payload after the envelope, little endian: u16 seq, u16 echoSeq, u16 echoHoldMs +static constexpr size_t k_probePayloadSize = 6; + +void PlayerConnection::ResetProbe() +{ + m_probeStats.reset(); + for (OutstandingProbe& probe : m_probeRing) + { + probe = OutstandingProbe(); + } + m_nextProbeSeq = 0; + m_lastPeerSeq = k_noProbeSeq; + m_lastPeerSeqRecvMs = 0; + m_nextProbeMs = 0; +} + +void PlayerConnection::TickProbe(int64_t nowMs) +{ + int pingIntervalMs = 250; + if (NGMP_OnlineServicesManager::GetInstance() != nullptr) + { + const ServiceConfig::Connection& connection = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig().connection; + pingIntervalMs = (TheNetwork != nullptr) ? connection.probe_match_interval_ms : connection.probe_lobby_interval_ms; + } + + // an echo rides on the next probe, so it is allowed one probe interval late + const int64_t timeoutMs = (std::max)(static_cast(1000), static_cast(m_probeStats.getLatency() * 4.0f)) + pingIntervalMs; + for (OutstandingProbe& probe : m_probeRing) + { + if (probe.seq != k_noProbeSeq && nowMs - probe.sendTimeMs > timeoutMs) + { + m_probeStats.noteRetry(static_cast(nowMs)); + probe.seq = k_noProbeSeq; + } + } + + m_probeStats.update(static_cast(nowMs)); + + if (nowMs < m_nextProbeMs || m_State != EConnectionState::CONNECTED_DIRECT) + { + return; + } + + m_nextProbeMs = nowMs + pingIntervalMs; + + if (AnticheatPlugInterface::DoesACPluginProvideSecureGameTransport()) + { + // the plugin transport has no per-peer receive path for the ping channel, so nothing could be echoed back + return; + } + + if (m_hSteamConnection == k_HSteamNetConnection_Invalid) + { + return; + } + + const uint16_t seq = m_nextProbeSeq; + m_nextProbeSeq = (m_nextProbeSeq == k_noProbeSeq - 1) ? 0 : static_cast(m_nextProbeSeq + 1); + + uint16_t echoHoldMs = 0; + if (m_lastPeerSeq != k_noProbeSeq) + { + echoHoldMs = static_cast((std::min)(nowMs - m_lastPeerSeqRecvMs, static_cast(k_noProbeSeq - 1))); + } + + BYTE packet[k_probePayloadSize]; + WriteU16(packet + 0, seq); + WriteU16(packet + 2, m_lastPeerSeq); + WriteU16(packet + 4, echoHoldMs); + + if (SendEnveloped(ENetworkChannel::NETWORK_CHANNEL_PING, packet, sizeof(packet), k_nSteamNetworkingSend_UnreliableNoNagle) != k_EResultOK) + { + return; + } + + // the slot is still in use: that probe was never echoed in a full ring turn + OutstandingProbe& slot = m_probeRing[seq % k_probeRingSize]; + if (slot.seq != k_noProbeSeq) + { + m_probeStats.noteRetry(static_cast(nowMs)); + } + slot.seq = seq; + slot.sendTimeMs = nowMs; +} + +int64_t PlayerConnection::GetArrivalMs(const SteamNetworkingMessage_t* msg) +{ + const int64_t nowMs = NowSteadyMs(); + if (msg == nullptr || SteamNetworkingUtils() == nullptr || msg->m_usecTimeReceived <= 0) + { + return nowMs; + } + + // how long the message waited in the transport's queue for the main loop + const int64_t queuedMs = (SteamNetworkingUtils()->GetLocalTimestamp() - msg->m_usecTimeReceived) / 1000; + return nowMs - (std::max)(static_cast(0), queuedMs); +} + +void PlayerConnection::HandleProbe(const uint8_t* payload, size_t len, int64_t arrivalMs) +{ + const size_t envelopeSize = (payload != nullptr) ? ConsumeEnvelope(payload, len) : 0; + if (envelopeSize == 0) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[PROBE] Dropping probe without a valid packet envelope from user %lld, size %zu", m_userID, len); + return; + } + payload += envelopeSize; + len -= envelopeSize; + + if (payload == nullptr || len < k_probePayloadSize) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[PROBE] Dropping malformed probe of size %zu from user %lld", len, m_userID); + return; + } + + // the arrival time, a slow frame on either side is no network delay + const int64_t nowMs = arrivalMs; + const uint16_t peerSeq = ReadU16(payload); + const uint16_t echoSeq = ReadU16(payload + 2); + const uint16_t echoHoldMs = ReadU16(payload + 4); + + // a reordered older probe must not move the echo back + if (peerSeq != k_noProbeSeq && (m_lastPeerSeq == k_noProbeSeq || static_cast(peerSeq - m_lastPeerSeq) < 0x8000)) + { + m_lastPeerSeq = peerSeq; + m_lastPeerSeqRecvMs = nowMs; + } + + if (echoSeq == k_noProbeSeq) + { + return; + } + + OutstandingProbe& acked = m_probeRing[echoSeq % k_probeRingSize]; + if (acked.seq != echoSeq) + { + // already resolved or timed out + return; + } + + const int64_t rttMs = (std::max)(static_cast(0), nowMs - acked.sendTimeMs - echoHoldMs); + m_probeStats.noteAcked(static_cast(rttMs), static_cast(nowMs)); + acked.seq = k_noProbeSeq; + + // older un-echoed probes are dropped, not lost: packet feedback measures the loss + for (OutstandingProbe& probe : m_probeRing) + { + if (probe.seq != k_noProbeSeq && static_cast(echoSeq - probe.seq) < 0x8000) + { + probe.seq = k_noProbeSeq; + } + } +} + PlayerConnection::PlayerConnection(int64_t userID, HSteamNetConnection hSteamConnection) { m_userID = userID; @@ -1327,57 +1496,14 @@ int PlayerConnection::SendGamePacket(void* pBuffer, uint32_t totalDataSize) return (int)k_EResultFail; } - ENetworkChannel netChannel = ENetworkChannel::NETWORK_CHANNEL_GAME; - std::vector vecData; - vecData.resize(totalDataSize + sizeof(ENetworkChannel)); - memcpy(vecData.data() + sizeof(ENetworkChannel), pBuffer, totalDataSize); - vecData[0] = (BYTE)netChannel; - int sendFlags = k_nSteamNetworkingSend_Reliable | k_nSteamNetworkingSend_AutoRestartBrokenSession; // default from last patch - ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - int netSendFlags = serviceConf.send.flags; - NetworkLog(ELogVerbosity::LOG_DEBUG, "[GAME PACKET] Sending msg of size %ld to user %lld\n", totalDataSize, m_userID); - EResult r = SteamNetworkingSockets()->SendMessageToConnection( - m_hSteamConnection, vecData.data(), vecData.size(), sendFlags, nullptr); - + const int r = SendEnveloped(ENetworkChannel::NETWORK_CHANNEL_GAME, static_cast(pBuffer), totalDataSize, sendFlags); if (r != k_EResultOK) { - if (netSendFlags != -1) - { - if (netSendFlags == 0) - { - sendFlags = k_nSteamNetworkingSend_Unreliable; - } - else if (netSendFlags == 1) - { - sendFlags = k_nSteamNetworkingSend_UnreliableNoNagle; - } - else if (netSendFlags == 2) - { - sendFlags = k_nSteamNetworkingSend_UnreliableNoDelay; - } - else if (netSendFlags == 3) - { - sendFlags = k_nSteamNetworkingSend_Reliable; - } - else if (netSendFlags == 4) - { - sendFlags = k_nSteamNetworkingSend_ReliableNoNagle; - } - } - - NetworkLog(ELogVerbosity::LOG_DEBUG, "[GAME PACKET] Sending msg of size %ld to user %lld\n", totalDataSize, m_userID); - EResult r = SteamNetworkingSockets()->SendMessageToConnection( - m_hSteamConnection, pBuffer, (int)totalDataSize, sendFlags, nullptr); - - if (r != k_EResultOK) - { - NetworkLog(ELogVerbosity::LOG_RELEASE, "[GAME PACKET] Failed to send, err code was %d", r); - } - - return (int)r; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[GAME PACKET] Failed to send, err code was %d", r); + return r; } } @@ -1428,8 +1554,8 @@ void PlayerConnection::UpdateLatencyHistogram() } - // update latency history - int currLatency = GetLatency(); + // update latency history, from the mesh probe once it has samples + int currLatency = (m_probeStats.getSampleCount() > 0) ? static_cast(m_probeStats.getLatency()) : GetLatency(); #if defined(GENERALS_ONLINE_HIGH_FPS_SERVER) const int connectionHistoryLength = histogram_duration / 16; // ~20 sec worth of frames at 60fps (default) #else @@ -1509,12 +1635,12 @@ bool PlayerConnection::IsIPV4() return info.m_addrRemote.IsIPv4(); } -int PlayerConnection::Recv(SteamNetworkingMessage_t** pMsg) +int PlayerConnection::Recv(SteamNetworkingMessage_t** pMsg, int maxMessages) { int r = -1; if (m_hSteamConnection != k_HSteamNetConnection_Invalid) { - r = SteamNetworkingSockets()->ReceiveMessagesOnConnection(m_hSteamConnection, pMsg, 255); + r = SteamNetworkingSockets()->ReceiveMessagesOnConnection(m_hSteamConnection, pMsg, maxMessages); NetworkLog(ELogVerbosity::LOG_DEBUG, "[DISC] Recv Result %d from user %lld", r, m_userID); } else @@ -1528,14 +1654,25 @@ int PlayerConnection::Recv(SteamNetworkingMessage_t** pMsg) std::string PlayerConnection::GetStats() { + // lead with the probe numbers, or just the reported ping while there are no samples yet + char szSummary[64] = { 0 }; + if (m_probeStats.getSampleCount() > 0) + { + snprintf(szSummary, sizeof(szSummary), "Lat %d j%.0f L%.0f%%\n", static_cast(m_probeStats.getLatency()), m_probeStats.getJitter(), m_probeStats.getLossPercent()); + } + else + { + snprintf(szSummary, sizeof(szSummary), "Lat %d\n", GetLatency()); + } + if (m_hSteamConnection == k_HSteamNetConnection_Invalid) - return "(disconnected)"; + return std::string(szSummary) + "(disconnected)"; char szBuf[2048] = { 0 }; int ret = SteamNetworkingSockets()->GetDetailedConnectionStatus(m_hSteamConnection, szBuf, 2048); NetworkLog(ELogVerbosity::LOG_RELEASE, "[STEAM] PlayerConnection::GetStats returned %d", ret); - return std::string(szBuf); + return std::string(szSummary) + szBuf; } @@ -1578,6 +1715,7 @@ void PlayerConnection::UpdateState(EConnectionState newState, NetworkMesh* pOwni { m_connectedSinceTime = std::chrono::steady_clock::now(); m_smoothedScore = -1.0f; + ResetProbe(); } m_State = newState; @@ -1688,6 +1826,51 @@ int PlayerConnection::GetLatency() return -1; } +size_t PlayerConnection::ConsumeEnvelope(const uint8_t* data, size_t len) +{ + const int64_t nowMs = NowSteadyMs(); + m_lastPacketMs = nowMs; + return m_feedback.ReadEnvelope(data, len, nowMs); +} + +int PlayerConnection::SendEnveloped(ENetworkChannel channel, const uint8_t* payload, size_t len, int sendFlags) +{ + const int64_t nowMs = NowSteadyMs(); + + std::vector vecData; + vecData.resize(sizeof(ENetworkChannel) + PacketFeedback::ENVELOPE_MAX_SIZE + len); + vecData[0] = static_cast(channel); + const size_t envelopeSize = m_feedback.WriteEnvelope(vecData.data() + sizeof(ENetworkChannel), nowMs); + memcpy(vecData.data() + sizeof(ENetworkChannel) + envelopeSize, payload, len); + vecData.resize(sizeof(ENetworkChannel) + envelopeSize + len); + + EResult r = SteamNetworkingSockets()->SendMessageToConnection(m_hSteamConnection, vecData.data(), static_cast(vecData.size()), sendFlags, nullptr); + if (r != k_EResultOK) + { + // once more with the same packet, it keeps its sequence number + r = SteamNetworkingSockets()->SendMessageToConnection(m_hSteamConnection, vecData.data(), static_cast(vecData.size()), sendFlags, nullptr); + } + if (r != k_EResultOK) + { + m_feedback.CancelEnvelope(); + } + return static_cast(r); +} + +bool PlayerConnection::GetOutgoingLoss(int windowMs, PacketFeedback::Summary& summary) +{ + const int64_t nowMs = NowSteadyMs(); + summary = m_feedback.GetOutgoing(nowMs, windowMs); + return summary.samples > 0; +} + +bool PlayerConnection::GetIncomingLoss(int windowMs, PacketFeedback::Summary& summary) +{ + const int64_t nowMs = NowSteadyMs(); + summary = m_feedback.GetIncoming(nowMs, windowMs); + return summary.samples > 0; +} + int PlayerConnection::GetJitter() { int sumDelta = 0; diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp index d6032342218..78fac2e4447 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp @@ -332,8 +332,26 @@ Bool NextGenTransport::doRecv(void) std::map& connections = pMesh->GetAllConnections(); for (auto& kvPair : connections) { + // Only take as many messages as there are free slots, the rest stay queued in GNS + int numFreeSlots = 0; + for (int iSlot = 0; iSlot < MAX_MESSAGES; ++iSlot) + { + if (!m_inBufferOccupied[iSlot] || m_inBuffer[iSlot].length == 0) + { + ++numFreeSlots; + } + } + + if (numFreeSlots == 0) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, + "Game Packet Recv: m_inBuffer is full, leaving packets from user %lld queued", + static_cast(kvPair.second.m_userID)); + continue; + } + SteamNetworkingMessage_t* pMsg[255] = { nullptr }; - int numPackets = kvPair.second.Recv(pMsg); + int numPackets = kvPair.second.Recv(pMsg, std::min(numFreeSlots, static_cast(std::size(pMsg)))); if (numPackets <= 0) continue; @@ -381,12 +399,28 @@ Bool NextGenTransport::doRecv(void) msg->Release(); continue; } + else if (netChannel == ENetworkChannel::NETWORK_CHANNEL_PING) + { + kvPair.second.HandleProbe(reinterpret_cast(vecPacketDataWithoutHeader.data()), vecPacketDataWithoutHeader.size(), PlayerConnection::GetArrivalMs(msg)); + msg->Release(); + continue; + } else if (netChannel == ENetworkChannel::NETWORK_CHANNEL_GAME) { NetworkLog(ELogVerbosity::LOG_DEBUG, "[GAME PACKET] Received message of size %u from user %lld", vecPacketDataWithoutHeader.size(), static_cast(kvPair.second.m_userID)); + // the packet envelope: sequence number and receiver feedback + const size_t envelopeSize = kvPair.second.ConsumeEnvelope(reinterpret_cast(vecPacketDataWithoutHeader.data()), vecPacketDataWithoutHeader.size()); + if (envelopeSize == 0) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Game Packet Recv: Dropping packet without a valid envelope from user %lld, probably another game version", static_cast(kvPair.second.m_userID)); + msg->Release(); + continue; + } + vecPacketDataWithoutHeader.erase(vecPacketDataWithoutHeader.begin(), vecPacketDataWithoutHeader.begin() + envelopeSize); + // Must at least contain the header if (vecPacketDataWithoutHeader.size() < sizeof(TransportMessageHeader)) { @@ -565,6 +599,12 @@ Bool NextGenTransport::doRecv(void) bRet = TRUE; } } + else + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Game Packet Recv: Dropping packet with unknown channel %u from user %lld", static_cast(vecDataWithHeader[0]), static_cast(kvPair.second.m_userID)); + msg->Release(); + continue; + } } else if (vecDataWithHeader.size() != -1) { diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/PacketFeedback.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/PacketFeedback.cpp new file mode 100644 index 00000000000..64c659e9799 --- /dev/null +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/PacketFeedback.cpp @@ -0,0 +1,179 @@ +#include "GameNetwork/GeneralsOnline/PacketFeedback.h" +#include "GameNetwork/GeneralsOnline/NetBytes.h" + +#include + +PacketFeedback::PacketFeedback() +{ + Reset(); +} + +void PacketFeedback::Reset() +{ + m_nextSequence = 0; + m_sentCount = 0; + m_haveHighest = false; + m_highest = 0; + m_mask = 0; + m_span = 0; + m_lastFeedbackMs = 0; + m_outgoing.clear(); + m_incoming.clear(); +} + +size_t PacketFeedback::WriteEnvelope(uint8_t* out, int64_t nowMs) +{ + WriteU16(out, m_nextSequence); + ++m_nextSequence; + ++m_sentCount; + out[2] = 0x80; + + if (!m_haveHighest || (m_lastFeedbackMs != 0 && nowMs - m_lastFeedbackMs < FEEDBACK_INTERVAL_MS)) + { + return ENVELOPE_BASE_SIZE; + } + + int received = 0; + int longestRun = 0; + int run = 0; + for (int bit = 0; bit < m_span; ++bit) + { + if (m_mask & (static_cast(1) << bit)) + { + ++received; + run = 0; + } + else + { + ++run; + longestRun = (std::max)(longestRun, run); + } + } + + out[2] |= 0x01; + WriteU16(out + 3, m_highest); + out[5] = m_span; + out[6] = static_cast(received); + out[7] = static_cast(longestRun); + + Report report; + report.timeMs = nowMs; + report.expected = m_span; + report.received = static_cast(received); + report.longestRun = static_cast(longestRun); + Push(m_incoming, report, nowMs); + + m_lastFeedbackMs = nowMs; + return ENVELOPE_MAX_SIZE; +} + +void PacketFeedback::CancelEnvelope() +{ + if (m_sentCount > 0) + { + --m_nextSequence; + --m_sentCount; + } +} + +size_t PacketFeedback::ReadEnvelope(const uint8_t* in, size_t len, int64_t nowMs) +{ + if (in == nullptr || len < ENVELOPE_BASE_SIZE || (in[2] & 0x80) == 0) + { + return 0; + } + + const bool hasFeedback = (in[2] & 0x01) != 0; + if (hasFeedback && len < ENVELOPE_MAX_SIZE) + { + return 0; + } + + const uint16_t sequence = ReadU16(in); + if (!m_haveHighest) + { + m_haveHighest = true; + m_highest = sequence; + m_mask = 1; + m_span = 1; + } + else + { + const int diff = static_cast(static_cast(sequence - m_highest)); + if (diff > 0) + { + m_mask = (diff >= 64) ? 0 : (m_mask << diff); + m_mask |= 1; + m_highest = sequence; + m_span = static_cast((std::min)(64, m_span + diff)); + } + else if (diff < 0 && -diff < 64) + { + m_mask |= static_cast(1) << (-diff); + } + } + + if (!hasFeedback) + { + return ENVELOPE_BASE_SIZE; + } + + Report report; + report.timeMs = nowMs; + report.expected = in[5]; + report.received = in[6]; + report.longestRun = in[7]; + if (report.expected > 64 || report.received > report.expected) + { + return ENVELOPE_MAX_SIZE; // implausible, the packet is still consumed + } + + // the peer cannot have expected more of our packets than we sent + report.expected = static_cast((std::min)(report.expected, m_sentCount)); + report.received = (std::min)(report.received, report.expected); + Push(m_outgoing, report, nowMs); + return ENVELOPE_MAX_SIZE; +} + +void PacketFeedback::Push(std::deque& reports, const Report& report, int64_t nowMs) +{ + reports.push_back(report); + while (!reports.empty() && nowMs - reports.front().timeMs > MAX_WINDOW_MS) + { + reports.pop_front(); + } +} + +PacketFeedback::Summary PacketFeedback::Summarize(const std::deque& reports, int64_t nowMs, int windowMs) +{ + Summary summary; + int expected = 0; + int received = 0; + for (const Report& report : reports) + { + if (nowMs - report.timeMs > windowMs) + { + continue; + } + ++summary.samples; + expected += report.expected; + received += report.received; + summary.longestRun = (std::max)(summary.longestRun, static_cast(report.longestRun)); + } + + if (expected > 0) + { + summary.lossPercent = 100.0f * static_cast(expected - received) / static_cast(expected); + } + return summary; +} + +PacketFeedback::Summary PacketFeedback::GetOutgoing(int64_t nowMs, int windowMs) const +{ + return Summarize(m_outgoing, nowMs, windowMs); +} + +PacketFeedback::Summary PacketFeedback::GetIncoming(int64_t nowMs, int windowMs) const +{ + return Summarize(m_incoming, nowMs, windowMs); +} diff --git a/GeneralsMD/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp b/GeneralsMD/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp index 5eda9635d85..e69e94431ab 100644 --- a/GeneralsMD/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp +++ b/GeneralsMD/Code/GameEngineDevice/Source/W3DDevice/GameClient/W3DDisplay.cpp @@ -1545,6 +1545,20 @@ void W3DDisplay::gatherDebugStats() // Network performance stats unibuffer.format(L"Run Ahead: %d, Net FPS: %d, Packet arrival cushion: %d", TheNetwork->getRunAhead(), TheNetwork->getFrameRate(), TheNetwork->getPacketArrivalCushion()); + + // per-connection latency, jitter and loss + for (Int slot = 0; slot < MAX_SLOTS; ++slot) { + const ConnectionStats* stats = TheNetwork->getConnectionStats(slot); + if (stats != nullptr && stats->getSampleCount() > 0) { + UnicodeString statsstr; + if (stats->getLatePercent() > 0.0f) { + statsstr.format(L" P%d %.0fms j%.1f l%.0f/%.0f%%", slot, stats->getLatency(), stats->getJitter(), stats->getLossPercent(), stats->getLatePercent()); + } else { + statsstr.format(L" P%d %.0fms j%.1f l%.0f%%", slot, stats->getLatency(), stats->getJitter(), stats->getLossPercent()); + } + unibuffer.concat(statsstr); + } + } m_displayStrings[NetStats]->setText( unibuffer ); // Client frame rate averages for all players in the game. This only works right for the packet router. From 37d350993dad80741a4baee132475cd7160356d5 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Thu, 22 Oct 2026 07:03:24 +0200 Subject: [PATCH 05/12] feat(online): Rate connections and show the network overlay --- .../Include/GameNetwork/ConnectionManager.h | 5 + .../Include/GameNetwork/ConnectionStats.h | 30 +- .../Include/GameNetwork/NetworkInterface.h | 1 + .../GameClient/MessageStream/CommandXlat.cpp | 3 +- .../Source/GameNetwork/ConnectionManager.cpp | 33 + .../Source/GameNetwork/ConnectionStats.cpp | 17 +- .../GameEngine/Source/GameNetwork/Network.cpp | 12 + GeneralsMD/Code/GameEngine/CMakeLists.txt | 2 + .../GameEngine/Include/GameClient/InGameUI.h | 1 + .../GeneralsOnline/ConnectionRating.h | 77 +++ .../GameNetwork/GeneralsOnline/NGMPGame.h | 13 + .../GameNetwork/GeneralsOnline/NetworkMesh.h | 12 +- .../GeneralsOnline/OnlineServices_Init.h | 7 +- .../GUICallbacks/Menus/WOLGameSetupMenu.cpp | 100 +-- .../GameEngine/Source/GameClient/InGameUI.cpp | 612 ++++++++++++++++-- .../GeneralsOnline/ConnectionRating.cpp | 275 ++++++++ .../GeneralsOnline/NetworkMesh.cpp | 161 +++-- .../GeneralsOnline/OnlineServices_Init.cpp | 2 +- 18 files changed, 1115 insertions(+), 248 deletions(-) create mode 100644 GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/ConnectionRating.h create mode 100644 GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/ConnectionRating.cpp diff --git a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h index d6df0d5b273..5812389f3f0 100644 --- a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h +++ b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h @@ -244,6 +244,11 @@ class ConnectionManager Bool m_stallActive; time_t m_stallStartTime; UnsignedInt m_stallMissingMask; + Int m_lastKnownRunAhead; + Int m_frameGroupingMs; +public: + Bool getOverlayInfo(Int slot, NetworkOverlayInfo &info); ///< Fills what the overlay and the connection rating need about one slot. +protected: #endif // ----------------------------------------------------------------------------- diff --git a/Core/GameEngine/Include/GameNetwork/ConnectionStats.h b/Core/GameEngine/Include/GameNetwork/ConnectionStats.h index ac93d274ea6..8e3789a1a43 100644 --- a/Core/GameEngine/Include/GameNetwork/ConnectionStats.h +++ b/Core/GameEngine/Include/GameNetwork/ConnectionStats.h @@ -134,7 +134,7 @@ class ArrivalCushionStats TimedRing m_samples; }; -// Frames the game wanted to advance but had to wait for commands, over the same sliding window. +// Frames the game wanted to advance but had to wait for commands, kept for a minute and counted over a window. class StallStats { public: @@ -144,12 +144,11 @@ class StallStats void note(Int durationMs, UnsignedInt missingSlotMask, time_t now); ///< One finished stall and the slots whose commands were missing when it began. void update(time_t now); - Int getCount() const { return m_events.size(); } - Int getTotalMs() const; - Int getCountForSlot(Int slot) const; + Int getCount(time_t now, Int windowMs) const; + Int getCountForSlot(Int slot, time_t now, Int windowMs) const; private: - enum { MAX_EVENTS = 512 }; + enum { MAX_EVENTS = 512, HISTORY_MS = 60000 }; struct Event { @@ -160,3 +159,24 @@ class StallStats TimedRing m_events; }; + +// What the network overlay and the connection rating need to know about one slot beyond its ConnectionStats. +struct NetworkOverlayInfo +{ + NetworkOverlayInfo() + : cushionSamples(0), cushionP5(0), cushionP50(0) + , stallsWindow(0), stallsMinute(0), stalledNow(FALSE), waitingOnSlot(FALSE) + , slotInBandMs(-1), runAhead(0), frameGroupingMs(0) {} + + Int cushionSamples; ///< Frames before execution that the frame commands of this slot arrived, over the stats window. + Int cushionP5; + Int cushionP50; + Int stallsWindow; ///< Stalls the local game had to wait on this slot for, over the stats window. + Int stallsMinute; ///< The same over the last minute. + Bool stalledNow; ///< The local game is waiting for commands right now. + Bool waitingOnSlot; ///< ... and this slot is one it is waiting on. + Int slotInBandMs; ///< In-band round trip this slot contributes to the run ahead, -1 if unknown. + + Int runAhead; ///< Frames. + Int frameGroupingMs; ///< The send interval. +}; diff --git a/Core/GameEngine/Include/GameNetwork/NetworkInterface.h b/Core/GameEngine/Include/GameNetwork/NetworkInterface.h index c8ff8f0122f..2e28de9e1d2 100644 --- a/Core/GameEngine/Include/GameNetwork/NetworkInterface.h +++ b/Core/GameEngine/Include/GameNetwork/NetworkInterface.h @@ -103,6 +103,7 @@ class NetworkInterface : public SubsystemInterface virtual const ConnectionStats* getConnectionStats(Int slot) const = 0; ///< Latency, jitter and loss of the connection to a slot, nullptr when there is none. #if defined(GENERALS_ONLINE) + virtual Bool getOverlayInfo(Int slot, NetworkOverlayInfo &info) = 0; virtual void SeedLatencyData(int highestLatency) = 0; virtual bool IsSlugging() = 0; #endif diff --git a/Core/GameEngine/Source/GameClient/MessageStream/CommandXlat.cpp b/Core/GameEngine/Source/GameClient/MessageStream/CommandXlat.cpp index 231a6f355b4..adb5bb72c8d 100644 --- a/Core/GameEngine/Source/GameClient/MessageStream/CommandXlat.cpp +++ b/Core/GameEngine/Source/GameClient/MessageStream/CommandXlat.cpp @@ -54,6 +54,7 @@ #include "Common/OptionPreferences.h" #include "GameClient/InGameUI.h" +#include "GameClient/Keyboard.h" #include "GameClient/CommandXlat.h" #include "GameClient/DebugDisplay.h" #include "GameClient/Drawable.h" @@ -3936,7 +3937,7 @@ GameMessageDisposition CommandTranslator::translateGameMessage(const GameMessage } else if (key == KEY_F10) { - NGMP_OnlineServicesManager::ToggleAdvancedNetworkStats(); + NGMP_OnlineServicesManager::ToggleNetworkOverlay((TheKeyboard != nullptr && TheKeyboard->isShift()) ? 2 : 1); disp = DESTROY_MESSAGE; } diff --git a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp index d569542efdb..abe73435b7a 100644 --- a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp +++ b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp @@ -343,6 +343,8 @@ void ConnectionManager::init() m_stallActive = FALSE; m_stallStartTime = 0; m_stallMissingMask = 0; + m_lastKnownRunAhead = 0; + m_frameGroupingMs = 0; #endif m_frameMetrics.init(); @@ -1412,6 +1414,10 @@ void ConnectionManager::updateRunAhead(Int oldRunAhead, Int frameRate, Bool didS static time_t lasttimesent = 0; time_t curTime = timeGetTime(); +#if defined(GENERALS_ONLINE) + m_lastKnownRunAhead = oldRunAhead; +#endif + if ((lasttimesent == 0) || ((curTime - lasttimesent) > TheGlobalData->m_networkRunAheadMetricsTime)) { if (m_localSlot == m_packetRouterSlot) { // We are the packet router, time to compute a new run ahead for this game. @@ -2031,6 +2037,9 @@ void ConnectionManager::setFrameGrouping(time_t frameGrouping) { if (m_localSlot == m_packetRouterSlot) { frameGrouping = frameGrouping / 2; } +#if defined(GENERALS_ONLINE) + m_frameGroupingMs = (Int)frameGrouping; +#endif for (Int i = 0; i < MAX_SLOTS; ++i) { if (m_connections[i] != nullptr) { m_connections[i]->setFrameGrouping(frameGrouping); @@ -2911,4 +2920,28 @@ void ConnectionManager::noteStalling(Bool stalling, UnsignedInt frame) { m_stallStats.note((Int)(now - m_stallStartTime), m_stallMissingMask, now); } } + +Bool ConnectionManager::getOverlayInfo(Int slot, NetworkOverlayInfo &info) { + time_t now = timeGetTime(); + m_stallStats.update(now); + + info = NetworkOverlayInfo(); + info.stalledNow = m_stallActive; + info.runAhead = m_lastKnownRunAhead; + info.frameGroupingMs = m_frameGroupingMs; + + if (slot >= 0 && slot < MAX_SLOTS) { + info.stallsWindow = m_stallStats.getCountForSlot(slot, now, NETWORK_STATS_WINDOW_MS); + info.stallsMinute = m_stallStats.getCountForSlot(slot, now, 60000); + info.waitingOnSlot = m_stallActive && ((m_stallMissingMask & (1u << slot)) != 0); + if (isPlayerConnected(slot)) { + info.slotInBandMs = (Int)(m_latencyAverages[slot] * 1000.0f); + } + m_cushionStats[slot].update(now); + info.cushionSamples = m_cushionStats[slot].getSampleCount(); + info.cushionP5 = m_cushionStats[slot].getPercentile(5); + info.cushionP50 = m_cushionStats[slot].getPercentile(50); + } + return TRUE; +} #endif diff --git a/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp b/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp index c0c99c3673a..08db5856949 100644 --- a/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp +++ b/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp @@ -289,25 +289,28 @@ void StallStats::note(Int durationMs, UnsignedInt missingSlotMask, time_t now) void StallStats::update(time_t now) { - m_events.prune(now, NETWORK_STATS_WINDOW_MS); + m_events.prune(now, HISTORY_MS); } -Int StallStats::getTotalMs() const +Int StallStats::getCount(time_t now, Int windowMs) const { - Int total = 0; + Int count = 0; for (Int i = 0; i < m_events.size(); ++i) { - total += m_events[i].ms; + if (now - m_events[i].time <= windowMs) + { + ++count; + } } - return total; + return count; } -Int StallStats::getCountForSlot(Int slot) const +Int StallStats::getCountForSlot(Int slot, time_t now, Int windowMs) const { Int count = 0; for (Int i = 0; i < m_events.size(); ++i) { - if (m_events[i].slotMask & (1u << slot)) + if ((m_events[i].slotMask & (1u << slot)) && now - m_events[i].time <= windowMs) { ++count; } diff --git a/Core/GameEngine/Source/GameNetwork/Network.cpp b/Core/GameEngine/Source/GameNetwork/Network.cpp index 9dc70133a97..d5e6b6ada93 100644 --- a/Core/GameEngine/Source/GameNetwork/Network.cpp +++ b/Core/GameEngine/Source/GameNetwork/Network.cpp @@ -166,6 +166,9 @@ class Network : public NetworkInterface virtual Int getAverageFPS() override { return m_conMgr->getAverageFPS(); } virtual Int getSlotAverageFPS(Int slot) override; virtual const ConnectionStats* getConnectionStats(Int slot) const override; +#if defined(GENERALS_ONLINE) + virtual Bool getOverlayInfo(Int slot, NetworkOverlayInfo &info) override; +#endif virtual void attachTransport(Transport *transport) override; virtual void initTransport() override; @@ -1108,6 +1111,15 @@ const ConnectionStats* Network::getConnectionStats(Int slot) const { return nullptr; } +#if defined(GENERALS_ONLINE) +Bool Network::getOverlayInfo(Int slot, NetworkOverlayInfo &info) { + if (m_conMgr != nullptr) { + return m_conMgr->getOverlayInfo(slot, info); + } + return FALSE; +} +#endif + #if defined(RTS_DEBUG) void Network::toggleNetworkOn() { if (m_networkOn == TRUE) { diff --git a/GeneralsMD/Code/GameEngine/CMakeLists.txt b/GeneralsMD/Code/GameEngine/CMakeLists.txt index 5bbf0e7c005..ee275be1383 100644 --- a/GeneralsMD/Code/GameEngine/CMakeLists.txt +++ b/GeneralsMD/Code/GameEngine/CMakeLists.txt @@ -1140,6 +1140,7 @@ set(GAMEENGINE_SRC Include/GameNetwork/GeneralsOnline/DiscordRichPresence.h Include/GameNetwork/GeneralsOnline/Vendor/DiscordRPC/discord_rpc.h Include/GameNetwork/GeneralsOnline/NetworkBitstream.h + Include/GameNetwork/GeneralsOnline/ConnectionRating.h Include/GameNetwork/GeneralsOnline/NetworkMesh.h Include/GameNetwork/GeneralsOnline/NetBytes.h Include/GameNetwork/GeneralsOnline/PacketFeedback.h @@ -1168,6 +1169,7 @@ set(GAMEENGINE_SRC Source/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.cpp Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp + Source/GameNetwork/GeneralsOnline/ConnectionRating.cpp Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp Source/GameNetwork/GeneralsOnline/PacketFeedback.cpp Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp diff --git a/GeneralsMD/Code/GameEngine/Include/GameClient/InGameUI.h b/GeneralsMD/Code/GameEngine/Include/GameClient/InGameUI.h index d5798a48ec7..872a4dab664 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameClient/InGameUI.h +++ b/GeneralsMD/Code/GameEngine/Include/GameClient/InGameUI.h @@ -615,6 +615,7 @@ class InGameUI : public SubsystemInterface, public Snapshot void drawRenderFps(Int& x, Int& y); void drawSystemTime(Int& x, Int& y); void drawGameTime(); + void drawNetworkOverlay(Int mode); ///< 1 = player view, 2 = dev view void drawPlayerInfoList(); void drawObserverStats(Int &x, Int &y); Bool m_observerStatsHidden = false; // hide/show observer overlay diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/ConnectionRating.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/ConnectionRating.h new file mode 100644 index 00000000000..1225735f622 --- /dev/null +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/ConnectionRating.h @@ -0,0 +1,77 @@ +#pragma once + +#include "Lib/BaseType.h" +#include "GameNetwork/ConnectionStats.h" + +// One overall rating of a connection, shared by the lobby slot indicator and the in-game overlay. +// It is the worse of two axes: delay (the input delay the link forces, in ms) and stability (stalls per minute). +// 5 bars: delay <= 150 ms and no stall in the last minute; 1 bar: delay above 500 ms or 6 or more stalls per minute. +// Colors: 4-5 green, 2-3 amber, 1 red. Red needs one of (RedReason): blackout, loss, delay or stalls, see RedReason in the .cpp. +// Stalls a healthy link does not explain wait on the peer's frame rate: slow PC, at most 3 bars and never red for it. +// Too few samples (10 probes, 20 command acks) give no rating. +class ConnectionRating +{ +public: + enum { BAR_COUNT = 5 }; + enum { SEVERITY_OK = 0, SEVERITY_WARN = 1, SEVERITY_BAD = 2 }; + + struct Inputs + { + Inputs(); + + Real delayMs; ///< Input delay the link forces. + Real stallsPerMinute; ///< Measured and predicted, whichever is higher. + Real measuredStallsPerMinute; ///< Stalls of the last minute in a match, negative in the lobby. + Real lossPercent; ///< Packet loss over the window from enough feedback reports, negative if unknown. + Int silentMs; ///< Since the last packet of the peer, negative if unknown. + Int silentLimitMs; ///< Silence that makes the link red, at least 2.5 probe intervals. + Bool stallInWindow; ///< A stall in the last 10 s stats window. + Bool slowPc; ///< The measured stalls are the peer's frame rate, not the link. + }; + + struct State + { + State() : bars(0), delayBars(0), stabilityBars(0), delayMs(0), stallsPerMinute(0.0f), raiseSinceMs(0), redReason(nullptr), slowPc(FALSE) {} + + Int bars; ///< 1..BAR_COUNT, 0 until rated. + Int delayBars; + Int stabilityBars; + Int delayMs; + Real stallsPerMinute; + UnsignedInt raiseSinceMs; ///< Since when the rating asks for more bars than shown, 0 if it does not. + const char *redReason; ///< Why the link is red, nullptr if it is not. + Bool slowPc; ///< The peer's PC is the bottleneck, the link is fine. + }; + + /// lossPercent is the packet loss from the packet feedback, negative to use the loss of the stats. + static Inputs InputsFromProbe(const ConnectionStats &probe, Int slackPercent, Int sendIntervalMs, Real lossPercent); + + static Inputs InputsFromMatch(const ConnectionStats &cmd, const NetworkOverlayInfo &info, Int slackPercent, Int sendIntervalMs, Real lossPercent); + + /// Fewer bars show at once, more only after the rating asked for them for a few seconds. + static void Update(State &state, const Inputs &inputs, UnsignedInt nowMs); + + static Int Severity(Int bars); + + static const char *RedReason(const Inputs &inputs); + + /// Draws the signal bars, level is bars - 1. + static void DrawBars(Int left, Int top, Int size, Int level); + + /// The spinner shown while a connection is not rated yet. + static void DrawSpinner(Int left, Int top, Int size); + + /// The indicator of a rating as ComputeConnectionRating returns it: the spinner below 1 bar, else the bars. + static void DrawIndicator(Int left, Int top, Int size, Int bars); +}; + +// Draws between begin and end of a display batch, unless the caller already batches. +class ScopedDisplayBatch +{ +public: + ScopedDisplayBatch(); + ~ScopedDisplayBatch(); + +private: + Bool m_ownBatch; +}; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMPGame.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMPGame.h index 0d11c6a51b3..1275b1bc5c8 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMPGame.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMPGame.h @@ -79,6 +79,19 @@ class NGMPGame : public GameInfo virtual ~NGMPGame(); virtual void reset(void); + Int getSlotOfUser(int64_t userID) const + { + for (Int slotNum = 0; slotNum < MAX_SLOTS; ++slotNum) + { + const NGMPGameSlot* pSlot = (const NGMPGameSlot*)getConstSlot(slotNum); + if (pSlot != nullptr && pSlot->m_userID == userID) + { + return slotNum; + } + } + return -1; + } + bool HasCommittedOutcome() const { return m_bHasCommittedOutcome; } void SetHasCommittedOutcome() { m_bHasCommittedOutcome = true; } diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h index 7bae0762a64..3faae1def69 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h @@ -8,6 +8,7 @@ #include "PluginInterfaces.h" #include "Lib/BaseType.h" #include "GameNetwork/ConnectionStats.h" +#include "GameNetwork/GeneralsOnline/ConnectionRating.h" #include "GameNetwork/GeneralsOnline/PacketFeedback.h" class NetRoom_ChatMessagePacket; @@ -92,7 +93,8 @@ class PlayerConnection } std::vector m_vecLatencyHistory; - std::vector m_vecQualityHistory; + float GetSustainedLossPercent(int windowMs); ///< The worse direction of the packet feedback loss from at least 3 reports, -1 if unknown. + std::string GetStats(); std::string GetConnectionType(); @@ -118,11 +120,13 @@ class PlayerConnection bool GetOutgoingLoss(int windowMs, PacketFeedback::Summary& summary); ///< our packets to the peer, false without feedback bool GetIncomingLoss(int windowMs, PacketFeedback::Summary& summary); ///< the peer's packets to us int GetJitter(); - float GetConnectionQuality(); - int ComputeConnectionScore(); + int ComputeConnectionRating(); ///< 1..5 bars, -1 while it is not known yet; cached for 250 ms. + int UpdateConnectionRating(); + int64_t m_nextRatingMs = 0; + int m_ratingResult = -1; std::chrono::steady_clock::time_point m_connectedSinceTime = (std::chrono::steady_clock::time_point::min)(); - float m_smoothedScore = -1.0f; + ConnectionRating::State m_rating; // Only set for Steam connections HSteamNetConnection m_hSteamConnection = k_HSteamNetConnection_Invalid; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index 9e64d15f6c4..87c12c5af1c 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -661,9 +661,10 @@ class NGMP_OnlineServicesManager static void CaptureScreenshot(bool bResizeForTransmit, std::function)> cbOnDataAvailable); static void CaptureScreenshotForProbe(EScreenshotType screenshotType, std::string strURI); - static bool g_bAdvancedNetworkStats; - static void ToggleAdvancedNetworkStats() { g_bAdvancedNetworkStats = !g_bAdvancedNetworkStats; } - static bool IsAdvancedNetworkStatsEnabled() { return g_bAdvancedNetworkStats; } + // the network overlay: 0 off, 1 player view (F10), 2 dev view (Shift+F10) + static int g_networkOverlayMode; + static void ToggleNetworkOverlay(int mode) { g_networkOverlayMode = (g_networkOverlayMode == mode) ? 0 : mode; } + static int GetNetworkOverlayMode() { return g_networkOverlayMode; } void OnLogin(ELoginResult loginResult, const char* szWSAddr, std::function fnWebsocketConnectedCallback); diff --git a/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp b/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp index 739367b2894..b04c4c1180d 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp @@ -262,10 +262,6 @@ enum }; static Int connectionIndicatorState[MAX_SLOTS]; -static const Int CONNECTION_LEVEL_COUNT = 5; -static const Int connectionLevelThresholds[CONNECTION_LEVEL_COUNT - 1] = { 40, 60, 75, 90 }; ///< score needed for 2..5 bars -static const Int CONNECTION_LEVEL_HYSTERESIS = 3; - WindowLayout *WOLMapSelectLayout = NULL; void PopBackToLobby() @@ -515,10 +511,11 @@ static void playerTooltip(GameWindow *window, } bool bIsConnected = false; - int connectionScore = -1; + int connectionBars = -1; int connectionLatency = -1; int connectionJitter = -1; - int connectionQualityPct = -1; + int connectionLossPct = -1; + int connectionDelayMs = -1; std::string strConnectionType = ""; LobbyMemberEntry member = pLobbyInterface->GetRoomMemberFromID(slot->m_userID); @@ -540,11 +537,12 @@ static void playerTooltip(GameWindow *window, { bIsConnected = false; } - connectionScore = pConnection->ComputeConnectionScore(); + connectionBars = pConnection->ComputeConnectionRating(); connectionLatency = pConnection->GetLatency(); connectionJitter = pConnection->GetJitter(); - float rawQuality = pConnection->GetConnectionQuality(); - connectionQualityPct = (rawQuality >= 0.0f) ? static_cast(rawQuality * 100.0f) : -1; + const float sustainedLoss = pConnection->GetSustainedLossPercent(60000); + connectionLossPct = (sustainedLoss >= 0.0f) ? static_cast(sustainedLoss + 0.5f) : -1; + connectionDelayMs = (connectionBars > 0) ? pConnection->m_rating.delayMs : -1; } } } @@ -562,19 +560,26 @@ static void playerTooltip(GameWindow *window, } else if (bIsConnected) { - UnicodeString scoreStr, latencyStr, jitterStr, qualityStr; - if (connectionScore >= 0) scoreStr.format(L"%d%%", connectionScore); else scoreStr = L"Unknown"; + UnicodeString latencyStr, jitterStr, qualityStr, delayStr; if (connectionLatency >= 0) latencyStr.format(L"%d ms", connectionLatency); else latencyStr = L"Unknown"; + if (connectionDelayMs >= 0) + { + delayStr.format(L"%d ms", connectionDelayMs); + } + else + { + delayStr = L"Unknown"; + } if (connectionJitter >= 0) jitterStr.format(L"%d ms", connectionJitter); else jitterStr = L"Unknown"; - if (connectionQualityPct >= 0) qualityStr.format(L"%d%%", connectionQualityPct); else qualityStr = L"Unknown"; + if (connectionLossPct >= 0) qualityStr.format(L"%d%%", connectionLossPct); else qualityStr = L"Unknown"; #if defined(WORLD_SERIES_CLIENT) - playerInfo.format(L"\nConnection State: Connected (%hs)\nConnection Score: %s\nLatency: %s\nJitter: %s\nReliability: %s\nRegion: %hs\nOverall Elo Rating: %d (in %d matches)\nWS Elo Rating: %d\nWins: %d\nLosses: %d\nDisconnects: %d\nFavorite Army: %s", - strConnectionType.c_str(), scoreStr.str(), latencyStr.str(), jitterStr.str(), qualityStr.str(), + playerInfo.format(L"\nConnection State: Connected (%hs)\nPing (round trip): %s\nPacket loss: %s\nJitter: %s\nExpected input delay: %s\nRegion: %hs\nOverall Elo Rating: %d (in %d matches)\nWS Elo Rating: %d\nWins: %d\nLosses: %d\nDisconnects: %d\nFavorite Army: %s", + strConnectionType.c_str(), latencyStr.str(), qualityStr.str(), jitterStr.str(), delayStr.str(), member.region.c_str(), stats.elo_rating, stats.elo_num_matches, stats.monthly_elo_rating, totalWins, totalLosses, totalDiscons, favoriteSide.str()); #else - playerInfo.format(L"\nConnection State: Connected (%hs)\nConnection Score: %s\nLatency: %s\nJitter: %s\nReliability: %s\nRegion: %hs\nOverall Elo Rating: %d (in %d matches)\nMonthly Elo Rating: %d\nWins: %d\nLosses: %d\nDisconnects: %d\nFavorite Army: %s", - strConnectionType.c_str(), scoreStr.str(), latencyStr.str(), jitterStr.str(), qualityStr.str(), + playerInfo.format(L"\nConnection State: Connected (%hs)\nPing (round trip): %s\nPacket loss: %s\nJitter: %s\nExpected input delay: %s\nRegion: %hs\nOverall Elo Rating: %d (in %d matches)\nMonthly Elo Rating: %d\nWins: %d\nLosses: %d\nDisconnects: %d\nFavorite Army: %s", + strConnectionType.c_str(), latencyStr.str(), qualityStr.str(), jitterStr.str(), delayStr.str(), member.region.c_str(), stats.elo_rating, stats.elo_num_matches, stats.monthly_elo_rating, totalWins, totalLosses, totalDiscons, favoriteSide.str()); #endif } @@ -1399,26 +1404,6 @@ void WOLDisplayGameOptions() } -// ----------------------------------------------------------------------------------------- -// The Bad munkee slot list displaying function -//------------------------------------------------------------------------------------------------- -static Int WOLUpdateConnectionLevel(Int level, Int score) -{ - if (level < 0) - { - level = 0; - while (level < CONNECTION_LEVEL_COUNT - 1 && score >= connectionLevelThresholds[level]) - ++level; - return level; - } - - while (level < CONNECTION_LEVEL_COUNT - 1 && score >= connectionLevelThresholds[level] + CONNECTION_LEVEL_HYSTERESIS) - ++level; - while (level > 0 && score < connectionLevelThresholds[level - 1] - CONNECTION_LEVEL_HYSTERESIS) - --level; - return level; -} - static void WOLRefreshConnectionIndicators(void) { NGMP_OnlineServices_LobbyInterface* pLobbyInterface = NGMP_OnlineServicesManager::GetInterface(); @@ -1445,7 +1430,7 @@ static void WOLRefreshConnectionIndicators(void) genericPingWindow[i]->winHide(FALSE); EConnectionState connectionState = EConnectionState::NOT_CONNECTED; - int connectionScore = -1; + int connectionBars = -1; if (pMesh != nullptr) { @@ -1453,17 +1438,17 @@ static void WOLRefreshConnectionIndicators(void) if (pConnection != nullptr) { connectionState = pConnection->GetState(); - connectionScore = pConnection->ComputeConnectionScore(); + connectionBars = pConnection->ComputeConnectionRating(); } } Int& state = connectionIndicatorState[i]; if (connectionState == EConnectionState::CONNECTION_FAILED || connectionState == EConnectionState::CONNECTION_DISCONNECTED) state = CONNECTION_INDICATOR_WARNING; - else if (connectionState != EConnectionState::CONNECTED_DIRECT || connectionScore < 0) + else if (connectionState != EConnectionState::CONNECTED_DIRECT || connectionBars < 0) state = CONNECTION_INDICATOR_CONNECTING; else - state = WOLUpdateConnectionLevel(state, connectionScore); + state = connectionBars - 1; // the rating is 1..5 bars, the state is the level 0..4 } } @@ -1484,6 +1469,8 @@ static void WOLConnectionIndicatorDraw(GameWindow *window, WinInstanceData *inst if (slot < 0) return; + ScopedDisplayBatch batch; + Int x, y, width, height; window->winGetScreenPosition(&x, &y); window->winGetSize(&width, &height); @@ -1492,26 +1479,11 @@ static void WOLConnectionIndicatorDraw(GameWindow *window, WinInstanceData *inst const Int left = x + (width - size) / 2; const Int top = y + (height - size) / 2; const Real centerX = left + size * 0.5f; - const Real centerY = top + size * 0.5f; const Int state = connectionIndicatorState[slot]; if (state == CONNECTION_INDICATOR_CONNECTING) { - // Faint full ring with a solid arc rotating around it, drawn as short segments. - const Int segmentCount = 24; - const Int arcSegments = 7; - const Real radius = size * 0.4f; - const Real rotation = (timeGetTime() % 900) * 2.0f * PI / 900.0f; - const Color ringColor = GameMakeColor(47, 55, 168, 70); // Zero Hour UI blue - const Color arcColor = GameMakeColor(47, 55, 168, 255); - for (Int segment = 0; segment < segmentCount; ++segment) - { - const Real startAngle = rotation + segment * 2.0f * PI / segmentCount; - const Real endAngle = startAngle + 2.0f * PI / segmentCount; - TheDisplay->drawLine((Int)(centerX + std::cos(startAngle) * radius), (Int)(centerY + std::sin(startAngle) * radius), - (Int)(centerX + std::cos(endAngle) * radius), (Int)(centerY + std::sin(endAngle) * radius), - 3.0f, segment < arcSegments ? arcColor : ringColor); - } + ConnectionRating::DrawSpinner(left, top, size); } else if (state == CONNECTION_INDICATOR_WARNING) { @@ -1528,21 +1500,7 @@ static void WOLConnectionIndicatorDraw(GameWindow *window, WinInstanceData *inst } else { - Color levelColor = GameMakeColor(220, 40, 40, 255); - if (state >= 3) - levelColor = GameMakeColor(0, 200, 0, 255); - else if (state == 2) - levelColor = GameMakeColor(255, 190, 0, 255); - const Color emptyColor = GameMakeColor(90, 90, 90, 200); - - const Int gap = 1; - const Int barWidth = max(1, (size - gap * (CONNECTION_LEVEL_COUNT - 1)) / CONNECTION_LEVEL_COUNT); - for (Int bar = 0; bar < CONNECTION_LEVEL_COUNT; ++bar) - { - const Int barHeight = max(1, size * (bar + 1) / CONNECTION_LEVEL_COUNT); - TheDisplay->drawFillRect(left + bar * (barWidth + gap), top + size - barHeight, barWidth, barHeight, - bar <= state ? levelColor : emptyColor); - } + ConnectionRating::DrawBars(left, top, size, state); } } diff --git a/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp b/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp index b833f4c59d1..68954303707 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp @@ -101,6 +101,11 @@ #include "GameNetwork/NetworkInterface.h" extern NetworkInterface * TheNetwork; #include + +namespace +{ + void overlayReleaseCache(); +} #endif @@ -2354,6 +2359,9 @@ void InGameUI::freeMessageResources() void InGameUI::freeCustomUiResources() { +#if defined(GENERALS_ONLINE) + overlayReleaseCache(); +#endif TheDisplayStringManager->freeDisplayString(m_networkLatencyString); m_networkLatencyString = nullptr; TheDisplayStringManager->freeDisplayString(m_renderFpsString); @@ -7171,99 +7179,561 @@ void InGameUI::drawSystemTime(Int& x, Int& y) } } -void InGameUI::drawGameTime() +#if defined(GENERALS_ONLINE) +namespace { - // draw connections - if (NGMP_OnlineServicesManager::IsAdvancedNetworkStatsEnabled()) + enum OverlayColumn + { + COL_SLOT, COL_NAME, COL_BARS, COL_PATH, COL_PROBE, COL_CMD, COL_JIT, COL_LOST, COL_LATE, + COL_CUSHION, COL_SAMPLES, COL_STALLS, COL_OUT, COL_IN, + COL_COUNT + }; + + const Int k_playerViewTop = 500; + const Real k_devViewTopFraction = 0.10f; // the dev table sits higher to clear the control bar + const Int k_nameMaxChars = 14; + const Int k_devViewHeaderLines = 2; + const Int k_devViewFooterLines = 1; + + const wchar_t *const s_overlayHeaders[COL_COUNT] = { - if (TheNGMPGame != nullptr) + L"Slot", L"Player", L"Bars", L"Path", L"Probe", L"Cmd", L"Jit", L"Lost%", L"Late%", + L"Cush5/50", L"n", L"Stalls", L"OutL%", L"InL%" + }; + + + struct OverlayRow + { + OverlayRow() : used(FALSE), bars(-1), slowPc(FALSE), direct(FALSE), pingMs(0), lossPercent(0.0f), latePercent(0.0f) { - NetworkMesh* pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + for (Int c = 0; c < COL_COUNT; ++c) + { + severity[c] = ConnectionRating::SEVERITY_OK; + } + } - NGMP_OnlineServices_LobbyInterface* pLobbyInterface = NGMP_OnlineServicesManager::GetInterface(); + Bool used; + UnicodeString name; + Int bars; + Bool slowPc; + Bool direct; + Int pingMs; + Real lossPercent; + Real latePercent; + NetworkOverlayInfo info; + UnicodeString cell[COL_COUNT]; + Int severity[COL_COUNT]; + }; - if (pMesh != nullptr && pLobbyInterface != nullptr) + Int overlayPercentSeverity(Real percent) + { + if (percent >= 5.0f) + { + return ConnectionRating::SEVERITY_BAD; + } + if (percent >= 0.5f) + { + return ConnectionRating::SEVERITY_WARN; + } + return ConnectionRating::SEVERITY_OK; + } + + Color overlayColor(Int severity) + { + if (severity >= ConnectionRating::SEVERITY_BAD) + { + return GameMakeColor(255, 90, 80, 255); + } + if (severity == ConnectionRating::SEVERITY_WARN) + { + return GameMakeColor(255, 195, 50, 255); + } + return GameMakeColor(235, 235, 235, 255); + } + + Color overlayDim() + { + return GameMakeColor(165, 165, 165, 255); + } + + // the overlay owns one DisplayString per text slot, so a slot whose text is unchanged keeps its built sentence and + // quads. a shared string would rebuild a sentence surface and texture on every draw, which is very slow on some drivers + const Int k_overlayRefreshMs = 250; + const Int k_strMeasure = 0; + const Int k_strHeaderBase = 1; + const Int k_strHeaderCount = 8; + const Int k_strRowBase = k_strHeaderBase + k_strHeaderCount; + const Int k_strRowCount = 8; + const Int k_strDevHeadBase = k_strRowBase + MAX_SLOTS * k_strRowCount; + const Int k_strDevCellBase = k_strDevHeadBase + COL_COUNT; + const Int k_strFooterBase = k_strDevCellBase + MAX_SLOTS * COL_COUNT; + const Int k_strFooterCount = 8; + const Int k_strTotal = k_strFooterBase + k_strFooterCount; + + struct OverlayCache + { + OverlayCache() : valid(FALSE), mode(0), refreshedMs(0), lineH(12), rowCount(0) + { + for (Int i = 0; i < k_strTotal; ++i) + { + strings[i] = nullptr; + } + for (Int c = 0; c < COL_COUNT; ++c) { - //std::vector& vecMembers = pLobbyInterface->GetMembersListForCurrentRoom(); + headerCell[c] = s_overlayHeaders[c]; + } + } - ISteamNetworkingSockets* pSteamNetSockets = SteamNetworkingSockets(); - if (!pSteamNetSockets) - { - return; - } + Bool valid; + Int mode; + UnsignedInt refreshedMs; + Int lineH; + Int rowCount; + NetworkOverlayInfo header; + UnicodeString waitingOn; + UnicodeString headerCell[COL_COUNT]; + OverlayRow rows[MAX_SLOTS]; + DisplayString *strings[k_strTotal]; + }; + + OverlayCache *s_overlayCache = nullptr; + GameFont *s_overlayFont = nullptr; + + void overlayReleaseCache() + { + if (s_overlayCache == nullptr) + { + return; + } + for (Int i = 0; i < k_strTotal; ++i) + { + if (s_overlayCache->strings[i] != nullptr && TheDisplayStringManager != nullptr) + { + TheDisplayStringManager->freeDisplayString(s_overlayCache->strings[i]); + } + } + delete s_overlayCache; + s_overlayCache = nullptr; + } + + DisplayString *overlayString(Int index) + { + DisplayString *&pString = s_overlayCache->strings[index]; + if (pString == nullptr) + { + pString = TheDisplayStringManager->newDisplayString(); + } + pString->setFont(s_overlayFont); + return pString; + } + + void overlaySetText(DisplayString *pString, const UnicodeString &text) + { + if (!(text == pString->getText())) + { + pString->setText(text); + } + } - int i = 0; - auto& allConnections = pMesh->GetAllConnections(); - if (allConnections.empty()) + Int overlayTextWidth(Int index, const UnicodeString &text) + { + DisplayString *pString = overlayString(index); + overlaySetText(pString, text); + return pString->getWidth(); + } + + Int overlayDrawText(Int index, Int x, Int y, const UnicodeString &text, Color color, Color dropColor) + { + DisplayString *pString = overlayString(index); + overlaySetText(pString, text); + pString->draw(x, y, color, dropColor); + return pString->getWidth(); + } + + Int overlayTableWidth(const Int *colWidth, const Bool *colShown, Int gap) + { + Int total = 0; + Int shown = 0; + for (Int c = 0; c < COL_COUNT; ++c) + { + if (colShown[c]) + { + total += colWidth[c]; + ++shown; + } + } + return total + (shown > 1 ? (shown - 1) * gap : 0); + } +} +#endif + +void InGameUI::drawNetworkOverlay(Int mode) +{ +#if defined(GENERALS_ONLINE) + if (TheNGMPGame == nullptr || TheNetwork == nullptr) + { + return; + } + + NetworkMesh* pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + NGMP_OnlineServices_LobbyInterface* pLobbyInterface = NGMP_OnlineServicesManager::GetInterface(); + if (pMesh == nullptr || pLobbyInterface == nullptr) + { + return; + } + + if (s_overlayCache == nullptr) + { + s_overlayCache = new OverlayCache(); + } + OverlayCache& cache = *s_overlayCache; + s_overlayFont = m_gameTimeString->getFont(); + + const Color dropColor = m_gameTimeDropColor; + const Color neutral = overlayColor(ConnectionRating::SEVERITY_OK); + const Color dim = overlayDim(); + + const UnsignedInt nowMs = timeGetTime(); + const Bool refresh = !cache.valid || cache.mode != mode || nowMs - cache.refreshedMs >= (UnsignedInt)k_overlayRefreshMs; + if (refresh) + { + cache.valid = TRUE; + cache.mode = mode; + cache.refreshedMs = nowMs; + + Int lineW, lineH; + DisplayString *pMeasure = overlayString(k_strMeasure); + overlaySetText(pMeasure, UnicodeString(L"Wg")); + pMeasure->getSize(&lineW, &lineH); + cache.lineH = (std::max)(lineH, 12); + TheNetwork->getOverlayInfo(-1, cache.header); + + for (Int slot = 0; slot < MAX_SLOTS; ++slot) + { + cache.rows[slot] = OverlayRow(); + } + cache.rowCount = 0; + cache.waitingOn.clear(); + OverlayRow (&rows)[MAX_SLOTS] = cache.rows; + Int& rowCount = cache.rowCount; + UnicodeString& waitingOn = cache.waitingOn; + + for (auto& connection : pMesh->GetAllConnections()) + { + PlayerConnection& conn = connection.second; + if (!conn.IsValid()) + { + continue; + } + + LobbyMemberEntry member = pLobbyInterface->GetRoomMemberFromID(connection.first); + if (member.display_name.empty() || member.m_SlotIndex < 0 || member.m_SlotIndex >= MAX_SLOTS) + { + continue; + } + + const Int slot = member.m_SlotIndex; + OverlayRow& row = rows[slot]; + row.used = TRUE; + row.name.set(from_utf8(member.display_name).c_str()); + row.bars = conn.ComputeConnectionRating(); + row.slowPc = conn.m_rating.slowPc; + row.direct = conn.IsDirect(); + row.pingMs = conn.GetLatency(); + ++rowCount; + + const ConnectionStats* pCmdStats = TheNetwork->getConnectionStats(slot); + const ConnectionStats& probeStats = conn.m_probeStats; + const Bool hasCmd = (pCmdStats != nullptr && pCmdStats->getSampleCount() > 0); + row.lossPercent = hasCmd ? pCmdStats->getLossPercent() : probeStats.getLossPercent(); + row.latePercent = hasCmd ? pCmdStats->getLatePercent() : 0.0f; + TheNetwork->getOverlayInfo(slot, row.info); + + if (row.info.waitingOnSlot) + { + if (!waitingOn.isEmpty()) { - return; + waitingOn.concat(L", "); } + waitingOn.concat(row.name); + } - for (auto& connection : allConnections) - { - if (!connection.second.IsValid()) - { - continue; - } + if (mode != 2) + { + continue; + } - LobbyMemberEntry lobbyMember = pLobbyInterface->GetRoomMemberFromID(connection.first); + UnicodeString* cell = row.cell; + Int* sev = row.severity; + cell[COL_SLOT].format(L"%d", slot); + cell[COL_NAME] = row.name; + cell[COL_NAME].truncateTo(k_nameMaxChars); + if (row.bars > 0) + { + cell[COL_BARS].format(L"%d", row.bars); + sev[COL_BARS] = ConnectionRating::Severity(row.bars); + } + else + { + cell[COL_BARS] = L"-"; + } + cell[COL_PATH].format(L"%hs %hs", conn.IsIPV4() ? "IPv4" : "IPv6", row.direct ? "direct" : "relay"); - const int k_nLanes = 1; - SteamNetConnectionRealTimeStatus_t status; - SteamNetConnectionRealTimeLaneStatus_t laneStatus[k_nLanes]; - - if (!TheNetwork) - { - continue; - } + if (probeStats.getSampleCount() > 0) + { + cell[COL_PROBE].format(L"%.0f", probeStats.getLatency()); + } + else + { + cell[COL_PROBE] = L"-"; + } - EResult res = pSteamNetSockets->GetConnectionRealTimeStatus(connection.second.m_hSteamConnection, &status, k_nLanes, laneStatus); + if (hasCmd) + { + const Real jitter = pCmdStats->getJitter(); + cell[COL_CMD].format(L"%.0f", pCmdStats->getLatency()); + cell[COL_JIT].format(L"%.1f", jitter); + sev[COL_JIT] = (jitter >= 60.0f) ? ConnectionRating::SEVERITY_BAD : ((jitter >= 30.0f) ? ConnectionRating::SEVERITY_WARN : ConnectionRating::SEVERITY_OK); + cell[COL_LOST].format(L"%.0f", pCmdStats->getLossPercent()); + sev[COL_LOST] = overlayPercentSeverity(pCmdStats->getLossPercent()); + cell[COL_LATE].format(L"%.0f", pCmdStats->getLatePercent()); + sev[COL_LATE] = overlayPercentSeverity(pCmdStats->getLatePercent()); + } + else + { + cell[COL_CMD] = L"-"; + cell[COL_JIT] = L"-"; + cell[COL_LOST] = L"-"; + cell[COL_LATE] = L"-"; + } - if (res == k_EResultNoConnection || lobbyMember.display_name.empty()) - { - continue; - } + if (row.info.cushionSamples > 0) + { + cell[COL_CUSHION].format(L"%d/%d", row.info.cushionP5, row.info.cushionP50); + sev[COL_CUSHION] = (row.info.cushionP5 <= 0) ? ConnectionRating::SEVERITY_BAD : ConnectionRating::SEVERITY_OK; + } + else + { + cell[COL_CUSHION] = L"-"; + } + cell[COL_SAMPLES].format(L"%d", row.info.cushionSamples); + cell[COL_STALLS].format(L"%d", row.info.stallsWindow); + sev[COL_STALLS] = (row.info.stallsWindow >= 3 && !row.slowPc) ? ConnectionRating::SEVERITY_BAD : ((row.info.stallsWindow > 0) ? ConnectionRating::SEVERITY_WARN : ConnectionRating::SEVERITY_OK); + + // out is how our packets fare to the peer, in is the peer's to us + PacketFeedback::Summary feedbackOut; + PacketFeedback::Summary feedbackIn; + if (conn.GetOutgoingLoss(NETWORK_STATS_WINDOW_MS, feedbackOut)) + { + cell[COL_OUT].format(L"%.1f", feedbackOut.lossPercent); + sev[COL_OUT] = overlayPercentSeverity(feedbackOut.lossPercent); + } + else + { + cell[COL_OUT] = L"-"; + } + if (conn.GetIncomingLoss(NETWORK_STATS_WINDOW_MS, feedbackIn)) + { + cell[COL_IN].format(L"%.1f", feedbackIn.lossPercent); + sev[COL_IN] = overlayPercentSeverity(feedbackIn.lossPercent); + } + else + { + cell[COL_IN] = L"-"; + } + } + } - int avgFPS = TheNetwork->getSlotAverageFPS(lobbyMember.m_SlotIndex); - - UnicodeString netString; - netString.format(L"\n[usr %s|%d][%hs %hs][AVGFPS: %d] Lat: %i, QL: %.2f, QR: %.2f OutP/s: %.2f, OutB/s: %.2f, InP/s: %.2f, InB/s: %.2f, SR %i PU: %d, PR: %d, NACK: %d, QT: %I64d", - from_utf8(lobbyMember.display_name).c_str(), - (int)res, - connection.second.IsIPV4() ? "IPv4" : "IPv6", - connection.second.IsDirect() ? "Direct" : "Relay", - avgFPS, - status.m_nPing, - status.m_flConnectionQualityLocal, - status.m_flConnectionQualityRemote, - status.m_flOutPacketsPerSec, - status.m_flOutBytesPerSec, - status.m_flInPacketsPerSec, - status.m_flInBytesPerSec, - status.m_nSendRateBytesPerSecond, - status.m_cbPendingUnreliable, - status.m_cbPendingReliable, - status.m_cbSentUnackedReliable, - status.m_usecQueueTime); - - int w, h; - m_gameTimeString->getSize(&w, &h); - - bool bIsHighQuality = true; - if (avgFPS < (GENERALS_ONLINE_HIGH_FPS_LIMIT - 1) || status.m_cbSentUnackedReliable >= 1000 || (status.m_flConnectionQualityLocal != -1.f && status.m_flConnectionQualityLocal < 1.f) || (status.m_flConnectionQualityRemote != -1.f && status.m_flConnectionQualityRemote < 1.f)) - { - bIsHighQuality = false; - } + const Int lineH = cache.lineH; + const Int segmentGap = (lineH * 3) / 4; + const Int displayWidth = TheDisplay->getWidth(); + const Int displayHeight = TheDisplay->getHeight(); - m_gameTimeString->setText(netString); - m_gameTimeString->draw(0, 500 + (i * h / 2), bIsHighQuality ? m_colorGood : m_colorBad, m_gameTimeDropColor); - ++i; - } + const Int left = 0; + + OverlayRow (&rows)[MAX_SLOTS] = cache.rows; + const Int rowCount = cache.rowCount; + const UnicodeString& waitingOn = cache.waitingOn; + const NetworkOverlayInfo& header = cache.header; + + const Int tableLines = (mode == 2) ? (k_devViewHeaderLines + rowCount + k_devViewFooterLines) : (1 + rowCount); + Int y = (mode == 2) ? (Int)(displayHeight * k_devViewTopFraction) : k_playerViewTop; + y = (std::min)(y, displayHeight - 4 - tableLines * lineH); + y = (std::max)(y, 0); + + const Int delayMs = header.runAhead * 1000 / GENERALS_ONLINE_HIGH_FPS_LIMIT; + { + Int x = left; + UnicodeString text; + text.format(L"Input delay %d ms", delayMs); + x += overlayDrawText(k_strHeaderBase + 0, x, y, text, header.stalledNow ? overlayColor(ConnectionRating::SEVERITY_BAD) : neutral, dropColor) + segmentGap; + + if (header.stalledNow && !waitingOn.isEmpty()) + { + text.format(L"waiting on: %ls", waitingOn.str()); + overlayDrawText(k_strHeaderBase + 3, x, y, text, overlayColor(ConnectionRating::SEVERITY_BAD), dropColor); + } + y += lineH; + } + + if (mode != 2) + { + Int nameWidth = 0; + for (Int slot = 0; slot < MAX_SLOTS; ++slot) + { + if (rows[slot].used) + { + nameWidth = (std::max)(nameWidth, overlayTextWidth(k_strRowBase + slot * k_strRowCount, rows[slot].name)); + } + } + + Int barRowY[MAX_SLOTS]; + Int barRowBars[MAX_SLOTS]; + Int barRowCount = 0; + for (Int slot = 0; slot < MAX_SLOTS; ++slot) + { + const OverlayRow& row = rows[slot]; + if (!row.used) + { + continue; + } + + // the indicators of all rows are drawn in one batch afterwards, text draws must not sit inside it + barRowY[barRowCount] = y; + barRowBars[barRowCount] = row.bars; + ++barRowCount; + + const Int rowStr = k_strRowBase + slot * k_strRowCount; + Int x = left + lineH + 8; + UnicodeString text; + overlayDrawText(rowStr, x, y, row.name, neutral, dropColor); + x += nameWidth + segmentGap; + text.format(L"ping %d ms", row.pingMs); + x += overlayDrawText(rowStr + 1, x, y, text, neutral, dropColor) + segmentGap; + text = row.direct ? L"direct" : L"relay"; + x += overlayDrawText(rowStr + 2, x, y, text, dim, dropColor) + segmentGap; + if (row.lossPercent >= 0.5f) + { + text.format(L"loss %.0f%%", row.lossPercent); + x += overlayDrawText(rowStr + 3, x, y, text, overlayColor(overlayPercentSeverity(row.lossPercent)), dropColor) + segmentGap; + } + if (row.latePercent >= 0.5f) + { + text.format(L"late %.0f%%", row.latePercent); + x += overlayDrawText(rowStr + 4, x, y, text, overlayColor(overlayPercentSeverity(row.latePercent)), dropColor) + segmentGap; + } + if (row.slowPc) + { + text = L"slow PC"; + x += overlayDrawText(rowStr + 5, x, y, text, overlayColor(ConnectionRating::SEVERITY_WARN), dropColor) + segmentGap; + } + else if (row.info.stallsWindow > 0) + { + text.format(L"stalls %d", row.info.stallsWindow); + x += overlayDrawText(rowStr + 5, x, y, text, overlayColor(row.info.stallsWindow >= 3 ? ConnectionRating::SEVERITY_BAD : ConnectionRating::SEVERITY_WARN), dropColor) + segmentGap; + } + y += lineH; + } + + if (barRowCount > 0) + { + ScopedDisplayBatch batch; + for (Int barRow = 0; barRow < barRowCount; ++barRow) + { + ConnectionRating::DrawIndicator(left, barRowY[barRow], lineH, barRowBars[barRow]); } + } + return; + } + // dev view: one row per peer, every column drawn at its own x as the font is proportional + Int colWidth[COL_COUNT]; + Bool colShown[COL_COUNT]; + Int colX[COL_COUNT]; + for (Int c = 0; c < COL_COUNT; ++c) + { + colShown[c] = TRUE; + colWidth[c] = overlayTextWidth(k_strDevHeadBase + c, cache.headerCell[c]); + for (Int slot = 0; slot < MAX_SLOTS; ++slot) + { + if (rows[slot].used) + { + colWidth[c] = (std::max)(colWidth[c], overlayTextWidth(k_strDevCellBase + slot * COL_COUNT + c, rows[slot].cell[c])); + } + } + } + + Int tableGap = segmentGap; + if (overlayTableWidth(colWidth, colShown, tableGap) > displayWidth - 8) + { + tableGap = (std::max)(4, lineH / 3); + } + const Int optionalColumns[3] = { COL_OUT, COL_IN, COL_BARS }; + for (Int o = 0; o < 3 && overlayTableWidth(colWidth, colShown, tableGap) > displayWidth - 8; ++o) + { + colShown[optionalColumns[o]] = FALSE; + } + + { + Int x = left; + for (Int c = 0; c < COL_COUNT; ++c) + { + colX[c] = x; + if (colShown[c]) + { + x += colWidth[c] + tableGap; + } } } + for (Int c = 0; c < COL_COUNT; ++c) + { + if (colShown[c]) + { + overlayDrawText(k_strDevHeadBase + c, colX[c], y, cache.headerCell[c], dim, dropColor); + } + } + y += lineH; + + for (Int slot = 0; slot < MAX_SLOTS; ++slot) + { + const OverlayRow& row = rows[slot]; + if (!row.used) + { + continue; + } + for (Int c = 0; c < COL_COUNT; ++c) + { + if (colShown[c]) + { + overlayDrawText(k_strDevCellBase + slot * COL_COUNT + c, colX[c], y, row.cell[c], overlayColor(row.severity[c]), dropColor); + } + } + y += lineH; + } + y += lineH / 2; + + { + UnicodeString text; + text.format(L"R %d frames (%d ms) send interval %d ms", header.runAhead, delayMs, header.frameGroupingMs); + overlayDrawText(k_strFooterBase + 0, left, y, text, neutral, dropColor); + } +#endif +} + +void InGameUI::drawGameTime() +{ + if (NGMP_OnlineServicesManager::GetNetworkOverlayMode() != 0) + { + drawNetworkOverlay(NGMP_OnlineServicesManager::GetNetworkOverlayMode()); + } +#if defined(GENERALS_ONLINE) + else + { + overlayReleaseCache(); + } +#endif + Int currentFrame = TheGameLogic->getFrame(); Int gameSeconds = (Int)(SECONDS_PER_LOGICFRAME_REAL * currentFrame); Int hours = gameSeconds / 60 / 60; diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/ConnectionRating.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/ConnectionRating.cpp new file mode 100644 index 00000000000..397c129250e --- /dev/null +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/ConnectionRating.cpp @@ -0,0 +1,275 @@ +#include "GameNetwork/GeneralsOnline/ConnectionRating.h" +#include "GameNetwork/NetworkDefs.h" +#include "GameClient/Color.h" +#include "GameClient/Display.h" + +#include +#include + +// upper limit of each band, index 0 is the limit for 5 bars, above the last is 1 bar +static const Real k_delayLimitsMs[ConnectionRating::BAR_COUNT - 1] = { 150.0f, 225.0f, 300.0f, 500.0f }; +static const Real k_stallLimitsPerMinute[ConnectionRating::BAR_COUNT - 1] = { 0.05f, 0.5f, 1.0f, 6.0f }; + +// Red (1 bar) only for a link that breaks the match, every other weakness is amber at worst: +static const Real k_redDelayMs = k_delayLimitsMs[ConnectionRating::BAR_COUNT - 2]; // input delay the run ahead would need +static const Real k_redStallsPerMinute = k_stallLimitsPerMinute[ConnectionRating::BAR_COUNT - 2]; // measured in a match, a prediction alone is never red +static const Real k_redLossPercent = 20.0f; // sustained packet loss, more than the redundancy repairs +static const Int k_redSilentMs = 1000; // no packet at all for this long, a blackout + +// Measured stalls of a link whose loss and tail predict almost none are the peer's slow frame rate +static const Real k_slowPcMinStalls = k_redStallsPerMinute; +static const Real k_slowPcMaxLinkStalls = 1.0f; +static const Int k_slowPcMaxBars = 3; + +static const Real k_hysteresis = 0.1f; // a band is left only when the value is this much past its limit + +static const UnsignedInt k_raiseHoldMs = 3000; // more bars are shown only after the rating asked for them this long + +// predicted stalls/min = 2.1 x loss% x 110 / (rtt + 60) + 0.1 x max(0, tail - max(100 ms, 0.2 x rtt)), tail = p90 round trip - smoothed round trip, up to 500 ms +static Real PredictStallsPerMinute(const ConnectionStats &stats, Real lossPercent) +{ + const Real tail = std::clamp(stats.getLatencyPercentile(90) - stats.getLatency(), 0.0f, 500.0f); + const Real headroom = std::max(100.0f, 0.2f * stats.getLatency()); + const Real excess = std::max(0.0f, tail - headroom); + const Real loss = (lossPercent >= 0.0f) ? lossPercent : stats.getRetryPercent(); + return 2.1f * loss * 110.0f / (stats.getLatency() + 60.0f) + 0.1f * excess; +} + +static Real EstimateDelayMs(Real roundTripMs, Int slackPercent, Int sendIntervalMs) +{ + return roundTripMs * (1.0f + slackPercent / 100.0f) + sendIntervalMs; +} + +ConnectionRating::Inputs::Inputs() + : delayMs(0.0f), stallsPerMinute(0.0f), measuredStallsPerMinute(-1.0f), lossPercent(-1.0f), silentMs(-1), silentLimitMs(k_redSilentMs), stallInWindow(FALSE), slowPc(FALSE) +{ +} + +ConnectionRating::Inputs ConnectionRating::InputsFromProbe(const ConnectionStats &probe, Int slackPercent, Int sendIntervalMs, Real lossPercent) +{ + Inputs inputs; + // the p75 round trip, not the smoothed one a few spikes pull up + inputs.delayMs = EstimateDelayMs(probe.getLatencyPercentile(75), slackPercent, sendIntervalMs); + inputs.stallsPerMinute = PredictStallsPerMinute(probe, lossPercent); + return inputs; +} + +ConnectionRating::Inputs ConnectionRating::InputsFromMatch(const ConnectionStats &cmd, const NetworkOverlayInfo &info, Int slackPercent, Int sendIntervalMs, Real lossPercent) +{ + Inputs inputs; + const Real roundTripMs = (info.slotInBandMs >= 0) ? (Real)info.slotInBandMs : cmd.getLatency(); + inputs.delayMs = EstimateDelayMs(roundTripMs, slackPercent, sendIntervalMs); + + Real stallsPerMinute = PredictStallsPerMinute(cmd, lossPercent); + if ((Real)info.stallsMinute >= k_slowPcMinStalls && stallsPerMinute < k_slowPcMaxLinkStalls) + { + inputs.stallsPerMinute = stallsPerMinute; + inputs.slowPc = TRUE; + return inputs; + } + if ((Real)info.stallsMinute > stallsPerMinute) + { + stallsPerMinute = (Real)info.stallsMinute; // the minute window holds one minute of stalls + } + inputs.stallsPerMinute = stallsPerMinute; + inputs.measuredStallsPerMinute = (Real)info.stallsMinute; + inputs.stallInWindow = info.stallsWindow > 0; + return inputs; +} + +static Int RateBand(Int previousBars, Real value, const Real *limits) +{ + const Int count = ConnectionRating::BAR_COUNT; + + Int raw = 1; + for (Int bars = count; bars >= 2; --bars) + { + if (value <= limits[count - bars]) + { + raw = bars; + break; + } + } + + if (previousBars < 1) + { + return raw; + } + + Int bars = previousBars; + while (bars < count && value <= limits[count - (bars + 1)] * (1.0f - k_hysteresis)) + { + ++bars; + } + while (bars > 1 && value > limits[count - bars] * (1.0f + k_hysteresis)) + { + --bars; + } + return bars; +} + +void ConnectionRating::Update(State &state, const Inputs &inputs, UnsignedInt nowMs) +{ + Real stalls = inputs.stallsPerMinute; + if (inputs.stallInWindow) + { + // a stall in the 10 s window rules out the two top bands + stalls = std::max(stalls, 0.6f); + } + else if (inputs.measuredStallsPerMinute > 0.0f) + { + // a stall in the last minute rules out the top band + stalls = std::max(stalls, 0.3f); + } + + state.delayBars = RateBand(state.delayBars, inputs.delayMs, k_delayLimitsMs); + state.stabilityBars = RateBand(state.stabilityBars, stalls, k_stallLimitsPerMinute); + Int rated = std::min(state.delayBars, state.stabilityBars); + state.redReason = RedReason(inputs); + if (state.redReason != nullptr) + { + rated = 1; + } + else if (rated < 2) + { + rated = 2; + } + state.slowPc = inputs.slowPc; + if (state.slowPc && rated > k_slowPcMaxBars) + { + rated = k_slowPcMaxBars; + } + if (state.bars < 1 || rated < state.bars) + { + state.bars = rated; + state.raiseSinceMs = 0; + } + else if (rated > state.bars) + { + // a noisy input crosses a limit back and forth, it must stay better for the whole hold + if (state.raiseSinceMs == 0) + { + state.raiseSinceMs = nowMs | 1; + } + else if (nowMs - state.raiseSinceMs >= k_raiseHoldMs) + { + state.bars = rated; + state.raiseSinceMs = 0; + } + } + else + { + state.raiseSinceMs = 0; + } + state.delayMs = static_cast(inputs.delayMs); + state.stallsPerMinute = stalls; +} + +const char *ConnectionRating::RedReason(const Inputs &inputs) +{ + if (inputs.silentMs >= inputs.silentLimitMs) + { + return "blackout"; + } + if (inputs.lossPercent >= k_redLossPercent) + { + return "loss"; + } + if (inputs.delayMs > k_redDelayMs) + { + return "delay"; + } + if (inputs.measuredStallsPerMinute >= k_redStallsPerMinute) + { + return "stalls"; + } + return nullptr; +} + +Int ConnectionRating::Severity(Int bars) +{ + if (bars >= 4) + { + return SEVERITY_OK; + } + if (bars >= 2) + { + return SEVERITY_WARN; + } + return SEVERITY_BAD; +} + +void ConnectionRating::DrawSpinner(Int left, Int top, Int size) +{ + // a faint full ring with a solid arc rotating around it, drawn as short segments + const Real twoPi = 6.28318531f; + const Int segmentCount = 24; + const Int arcSegments = 7; + const Real radius = size * 0.4f; + const Real centerX = left + size * 0.5f; + const Real centerY = top + size * 0.5f; + const Real rotation = (timeGetTime() % 900) * twoPi / 900.0f; + const Color ringColor = GameMakeColor(47, 55, 168, 70); // Zero Hour UI blue + const Color arcColor = GameMakeColor(47, 55, 168, 255); + for (Int segment = 0; segment < segmentCount; ++segment) + { + const Real startAngle = rotation + segment * twoPi / segmentCount; + const Real endAngle = startAngle + twoPi / segmentCount; + TheDisplay->drawLine((Int)(centerX + std::cos(startAngle) * radius), (Int)(centerY + std::sin(startAngle) * radius), + (Int)(centerX + std::cos(endAngle) * radius), (Int)(centerY + std::sin(endAngle) * radius), + 3.0f, segment < arcSegments ? arcColor : ringColor); + } +} + +void ConnectionRating::DrawIndicator(Int left, Int top, Int size, Int bars) +{ + if (bars < 1) + { + DrawSpinner(left, top, size); + } + else + { + DrawBars(left, top, size, bars - 1); + } +} + +ScopedDisplayBatch::ScopedDisplayBatch() + : m_ownBatch(!TheDisplay->isBatching()) +{ + if (m_ownBatch) + { + TheDisplay->beginBatch(); + } +} + +ScopedDisplayBatch::~ScopedDisplayBatch() +{ + if (m_ownBatch) + { + TheDisplay->endBatch(); + } +} + +void ConnectionRating::DrawBars(Int left, Int top, Int size, Int level) +{ + Color levelColor = GameMakeColor(220, 40, 40, 255); + const Int severity = Severity(level + 1); + if (severity == SEVERITY_OK) + { + levelColor = GameMakeColor(0, 200, 0, 255); + } + else if (severity == SEVERITY_WARN) + { + levelColor = GameMakeColor(255, 190, 0, 255); + } + const Color emptyColor = GameMakeColor(90, 90, 90, 200); + + const Int gap = 1; + const Int barWidth = std::max(1, (size - gap * (BAR_COUNT - 1)) / BAR_COUNT); + for (Int bar = 0; bar < BAR_COUNT; ++bar) + { + const Int barHeight = std::max(1, size * (bar + 1) / BAR_COUNT); + TheDisplay->drawFillRect(left + bar * (barWidth + gap), top + size - barHeight, barWidth, barHeight, + bar <= level ? levelColor : emptyColor); + } +} diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp index 25048c38a0f..d1890a35bb7 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp @@ -6,6 +6,7 @@ #include #include "GameNetwork/NetworkDefs.h" #include "GameNetwork/NetworkInterface.h" +#include "Common/GlobalData.h" #include "GameLogic/GameLogic.h" #include "../OnlineServices_RoomsInterface.h" #include "../json.hpp" @@ -1567,38 +1568,6 @@ void PlayerConnection::UpdateLatencyHistogram() m_vecLatencyHistory.erase(m_vecLatencyHistory.begin()); } m_vecLatencyHistory.push_back(currLatency); - - // Sample connection quality into rolling history. - // Prefer SNS local quality, then remote quality, then fall back to manual in/out packet rate ratio. - if (m_hSteamConnection != k_HSteamNetConnection_Invalid) - { - SteamNetConnectionRealTimeStatus_t status; - if (SteamNetworkingSockets()->GetConnectionRealTimeStatus(m_hSteamConnection, &status, 0, nullptr) == k_EResultOK) - { - float sample = -1.0f; - if (status.m_flConnectionQualityLocal >= 0.0f) - { - sample = status.m_flConnectionQualityLocal; - } - else if (status.m_flConnectionQualityRemote >= 0.0f) - { - sample = status.m_flConnectionQualityRemote; - } - else if (status.m_flOutPacketsPerSec > 0.0f) - { - sample = status.m_flInPacketsPerSec / status.m_flOutPacketsPerSec; - } - - if (sample > 1.0f) sample = 1.0f; - - if (sample >= 0.0f) - { - if (m_vecQualityHistory.size() >= connectionHistoryLength) - m_vecQualityHistory.erase(m_vecQualityHistory.begin()); - m_vecQualityHistory.push_back(sample); - } - } - } } void PlayerConnection::Close() @@ -1714,7 +1683,8 @@ void PlayerConnection::UpdateState(EConnectionState newState, NetworkMesh* pOwni if (newState == EConnectionState::CONNECTED_DIRECT && m_State != EConnectionState::CONNECTED_DIRECT) { m_connectedSinceTime = std::chrono::steady_clock::now(); - m_smoothedScore = -1.0f; + m_rating = ConnectionRating::State(); + m_nextRatingMs = 0; ResetProbe(); } @@ -1871,6 +1841,21 @@ bool PlayerConnection::GetIncomingLoss(int windowMs, PacketFeedback::Summary& su return summary.samples > 0; } +float PlayerConnection::GetSustainedLossPercent(int windowMs) +{ + // one report of a short window is no trend + static constexpr int k_minFeedbackReports = 3; + PacketFeedback::Summary outLoss; + PacketFeedback::Summary inLoss; + GetOutgoingLoss(windowMs, outLoss); + GetIncomingLoss(windowMs, inLoss); + if (outLoss.samples < k_minFeedbackReports && inLoss.samples < k_minFeedbackReports) + { + return -1.0f; + } + return (std::max)((outLoss.samples >= k_minFeedbackReports) ? outLoss.lossPercent : 0.0f, (inLoss.samples >= k_minFeedbackReports) ? inLoss.lossPercent : 0.0f); +} + int PlayerConnection::GetJitter() { int sumDelta = 0; @@ -1899,20 +1884,20 @@ int PlayerConnection::GetJitter() return sumDelta / count; } -float PlayerConnection::GetConnectionQuality() +int PlayerConnection::ComputeConnectionRating() { - if (!m_vecQualityHistory.empty()) + // the overlay and the lobby ask every frame, the percentile sorts inside do not change that fast + const int64_t ratingNowMs = NowSteadyMs(); + if (ratingNowMs < m_nextRatingMs) { - float sum = 0.0f; - for (float r : m_vecQualityHistory) - sum += r; - return sum / static_cast(m_vecQualityHistory.size()); + return m_ratingResult; } - - return -1.0f; + m_nextRatingMs = ratingNowMs + 250; + m_ratingResult = UpdateConnectionRating(); + return m_ratingResult; } -int PlayerConnection::ComputeConnectionScore() +int PlayerConnection::UpdateConnectionRating() { static constexpr std::chrono::milliseconds k_warmupDuration{ 1000 }; if (m_connectedSinceTime == (std::chrono::steady_clock::time_point::min)() || @@ -1921,65 +1906,71 @@ int PlayerConnection::ComputeConnectionScore() return -1; } - // TODO_EOS: need to impl jitter etc again - const int latency = GetLatency(); - const int jitter = GetJitter(); - const float quality = GetConnectionQuality(); // packet delivery ratio [0..1] + if (NGMP_OnlineServicesManager::GetInstance() == nullptr) + { + return -1; + } + const ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); + const int slackPercent = serviceConf.input_delay.slack_percent; + int sendIntervalMs = serviceConf.send.frames_per_packet * 1000 / GENERALS_ONLINE_HIGH_FPS_LIMIT; - static constexpr float k_subScoreFloor = 0.3f; - static constexpr float k_latencyWeight = 0.30f; - static constexpr float k_jitterWeight = 0.25f; - static constexpr float k_reliabilityWeight = 0.45f; + // the command connections are indexed by game slot, find the slot of this user + int slot = -1; + const ConnectionStats* pCmd = nullptr; + if (TheNetwork != nullptr && TheNGMPGame != nullptr) + { + slot = TheNGMPGame->getSlotOfUser(m_userID); + pCmd = TheNetwork->getConnectionStats(slot); + } - float weightedLogSum = 0.0f; - float activeWeightSum = 0.0f; + ConnectionRating::Inputs inputs; + NetworkOverlayInfo info; - if (latency >= 0) + float feedbackLoss = -1.0f; + static constexpr int k_minCmdSamples = 20; + const bool inMatch = (pCmd != nullptr && pCmd->getSampleCount() >= k_minCmdSamples); + const int feedbackWindowMs = inMatch ? NETWORK_STATS_WINDOW_MS : 60000; + PacketFeedback::Summary outLoss; + PacketFeedback::Summary inLoss; + const bool hasOut = GetOutgoingLoss(feedbackWindowMs, outLoss); + const bool hasIn = GetIncomingLoss(feedbackWindowMs, inLoss); + if (hasOut || hasIn) { - // 10ms and below are treated as full score. Above that, roughly: - // 400ms -> composite 75, 800ms -> composite 50 when other metrics are perfect. - int effectiveLatency = (std::max)(latency - 10, 0); - float latFactor = std::clamp(1.0f - static_cast(effectiveLatency) / 1590.0f, 0.0f, 1.0f); - float latencyScore = (std::max)(std::powf(latFactor, 4.545f), k_subScoreFloor); - weightedLogSum += k_latencyWeight * std::logf(latencyScore); - activeWeightSum += k_latencyWeight; + feedbackLoss = (std::max)(hasOut ? outLoss.lossPercent : 0.0f, hasIn ? inLoss.lossPercent : 0.0f); } + const float sustainedLoss = GetSustainedLossPercent(feedbackWindowMs); - if (jitter >= 0) + if (inMatch && TheNetwork->getOverlayInfo(slot, info)) { - // 50ms -> composite 75, 100ms -> composite 50 when other metrics are perfect. - float jitFactor = std::clamp(1.0f - static_cast(jitter) / 200.0f, 0.0f, 1.0f); - float jitterScore = (std::max)(std::powf(jitFactor, 2.632f), k_subScoreFloor); - weightedLogSum += k_jitterWeight * std::logf(jitterScore); - activeWeightSum += k_jitterWeight; + if (info.frameGroupingMs > 0) + { + sendIntervalMs = info.frameGroupingMs; + } + inputs = ConnectionRating::InputsFromMatch(*pCmd, info, slackPercent, sendIntervalMs, feedbackLoss); } - - if (quality >= 0.0f) + else if (m_probeStats.getSampleCount() >= 10) { - // 90% -> composite 75, 80% -> composite 50 when other metrics are perfect. - float relFactor = std::clamp(2.5f * quality - 1.5f, 0.0f, 1.0f); - float reliabilityScore = (std::max)(std::powf(relFactor, 2.5f), k_subScoreFloor); - weightedLogSum += k_reliabilityWeight * std::logf(reliabilityScore); - activeWeightSum += k_reliabilityWeight; + inputs = ConnectionRating::InputsFromProbe(m_probeStats, slackPercent, sendIntervalMs, feedbackLoss); } - - if (activeWeightSum <= 0.0f) + else { - return -1; + // a gap in the samples keeps the last rating, the indicator must not blink out + return (m_rating.bars > 0) ? m_rating.bars : -1; } - float composite = std::expf(weightedLogSum / activeWeightSum); - float rawScore = composite * 100.0f; - - static constexpr float k_scoreSmoothingAlpha = 0.15f; - if (m_smoothedScore < 0.0f) + inputs.lossPercent = sustainedLoss; + if (m_lastPacketMs > 0) { - m_smoothedScore = rawScore; + const int64_t nowSteadyMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + inputs.silentMs = static_cast((std::min)(nowSteadyMs - m_lastPacketMs, static_cast(3600000))); } - else { - m_smoothedScore += k_scoreSmoothingAlpha * (rawScore - m_smoothedScore); + // silence must outlast the probe interval of the current phase, probes may be the only traffic + const ServiceConfig::Connection& pingConfig = serviceConf.connection; + const int pingIntervalMs = (TheNetwork != nullptr) ? pingConfig.probe_match_interval_ms : pingConfig.probe_lobby_interval_ms; + inputs.silentLimitMs = (std::max)(inputs.silentLimitMs, pingIntervalMs * 5 / 2); } - return static_cast(std::round(m_smoothedScore)); + ConnectionRating::Update(m_rating, inputs, timeGetTime()); + return m_rating.bars; } diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp index 46a35dfed6f..29ae73bc5bd 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp @@ -42,7 +42,7 @@ std::mutex NGMP_OnlineServicesManager::m_ScreenshotMutex; std::vector NGMP_OnlineServicesManager::m_vecGuardedSSData; -bool NGMP_OnlineServicesManager::g_bAdvancedNetworkStats; +int NGMP_OnlineServicesManager::g_networkOverlayMode; NetworkMesh* NGMP_OnlineServicesManager::GetNetworkMesh() { From 6ecc67e4bbfe320086ba2093c2eda1cde1819afb Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Thu, 22 Oct 2026 07:03:26 +0200 Subject: [PATCH 06/12] feat(network): Repair datagram loss with adaptive redundancy --- .../Include/GameNetwork/Connection.h | 32 ++ .../Include/GameNetwork/ConnectionManager.h | 13 + .../Include/GameNetwork/ConnectionStats.h | 8 + .../Include/GameNetwork/FrameData.h | 2 +- .../Include/GameNetwork/FrameDataManager.h | 2 +- .../Include/GameNetwork/NetCommandRef.h | 41 +- .../GameNetwork/NetCommandWrapperList.h | 12 + .../Include/GameNetwork/NetworkSettings.h | 8 + .../Source/GameNetwork/Connection.cpp | 355 +++++++++++++++++- .../Source/GameNetwork/ConnectionManager.cpp | 92 ++++- .../Source/GameNetwork/ConnectionStats.cpp | 31 ++ .../Source/GameNetwork/FrameData.cpp | 5 +- .../Source/GameNetwork/FrameDataManager.cpp | 5 +- .../Source/GameNetwork/NetCommandRef.cpp | 3 + .../GameNetwork/NetCommandWrapperList.cpp | 34 ++ .../Source/GameNetwork/NetPacket.cpp | 3 + .../GeneralsOnline/OnlineServices_Init.h | 9 + .../GameEngine/Source/GameClient/InGameUI.cpp | 7 +- .../GeneralsOnline/NetworkMesh.cpp | 17 +- 19 files changed, 656 insertions(+), 23 deletions(-) diff --git a/Core/GameEngine/Include/GameNetwork/Connection.h b/Core/GameEngine/Include/GameNetwork/Connection.h index 0c77332ab1b..c1f409e3c7e 100644 --- a/Core/GameEngine/Include/GameNetwork/Connection.h +++ b/Core/GameEngine/Include/GameNetwork/Connection.h @@ -40,6 +40,9 @@ #include "GameNetwork/Transport.h" #include "GameNetwork/NetPacket.h" #include "GameNetwork/ConnectionStats.h" +#if defined(GENERALS_ONLINE) +#include "GameNetwork/NetworkSettings.h" +#endif class Connection : public MemoryPoolObject { @@ -61,6 +64,12 @@ class Connection : public MemoryPoolObject void setUser(User *user); User *getUser(); void setFrameGrouping(time_t frameGrouping); +#if defined(GENERALS_ONLINE) + void setNetworkSettings(const NetworkSettings &settings, Bool unreliableTransport) { m_networkSettings = settings; m_unreliableTransport = unreliableTransport; } +#endif + void setSlot(Int slot) { m_slot = slot; } + Int getRedundancyDepth() const { return m_redundancyDepth; } + time_t getFrameGrouping() const { return m_frameGrouping; } ///< The send interval in use, shorter than the configured one while redundancy is on. void sendNetCommandMsg(NetCommandMsg *msg, UnsignedByte relay); @@ -82,6 +91,21 @@ class Connection : public MemoryPoolObject #endif protected: +#if defined(GENERALS_ONLINE) + NetworkSettings m_networkSettings; + Bool m_unreliableTransport; +#endif + + void resetRedundancy(); + void updateRedundancy(time_t curtime, Bool &active, time_t &window, Int &depth); + Bool hasLossEvidence(); ///< A packet was lost in either direction lately, or nothing says otherwise + Bool getOutgoingLoss(float &lossPercent, Int &longestRun); ///< FALSE when the peer sent no packet feedback + void noteLossEvent(time_t now, Bool burst); + void setRedundancyDepth(Int depth, Int cap, time_t now); + Bool isQueueBuilding(time_t now); ///< Round trip at 2x its low while packets are lost: a full queue + void applyFrameGrouping(); + void addRedundantCommands(NetPacket *packet, time_t curtime, time_t window, Int depth); + Bool m_isQuitting; UnsignedInt m_quitTime; @@ -92,6 +116,14 @@ class Connection : public MemoryPoolObject time_t m_retryTime; ///< The time between sending retry packets for this connection. Time is in milliseconds. ConnectionStats m_stats; ///< Latency, jitter and loss of this connection. + Int m_redundancyDepth; ///< How many later packets each un-acked command is repeated in, 0 on a clean connection. + Int m_redundancyCap; ///< The deepest depth, from the longest burst to repair and the send interval. + time_t m_lastLossEventTime; ///< What the decay counts from. + time_t m_lastDepthChangeTime; ///< A burst needs a round trip to show whether the new depth helped. + Bool m_queueSuspected; ///< A full queue on the path, redundancy steps down instead of up. + Int m_slot; + + time_t m_configuredFrameGrouping; time_t m_frameGrouping; ///< The minimum time between packet sends. time_t m_lastTimeSent; ///< The time of the last packet send. }; diff --git a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h index 5812389f3f0..c5bee5502f2 100644 --- a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h +++ b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h @@ -239,6 +239,19 @@ class ConnectionManager Bool m_didSelfSlug; #if defined(GENERALS_ONLINE) + // the last command ids seen per sender, so a redundant copy of a command that is not synchronized is dropped + enum { RECENT_ID_COUNT = 256 }; + struct RecentCommandIds + { + Bool valid; + UnsignedShort highest; + Bool seen[RECENT_ID_COUNT]; + }; + RecentCommandIds m_recentIds[MAX_SLOTS]; + + Bool isDuplicateCommand(NetCommandMsg *msg); + void resetRecentCommandIds(); + ArrivalCushionStats m_cushionStats[MAX_SLOTS]; StallStats m_stallStats; Bool m_stallActive; diff --git a/Core/GameEngine/Include/GameNetwork/ConnectionStats.h b/Core/GameEngine/Include/GameNetwork/ConnectionStats.h index 8e3789a1a43..db57961c0d0 100644 --- a/Core/GameEngine/Include/GameNetwork/ConnectionStats.h +++ b/Core/GameEngine/Include/GameNetwork/ConnectionStats.h @@ -67,17 +67,22 @@ class ConnectionStats void noteAcked(Real rttMs, time_t now); ///< A command that was acked on its first send. void noteRetry(time_t now); ///< A command that was not acked in time and had to be sent again. void noteLateAck(time_t now); ///< A retried command was acked by its earlier send after all, the retry was not a loss. + void setRedundancyDepth(Int depth, Int cap); void update(time_t now); ///< Drop the samples that have left the window. Real getLatency() const { return m_srtt; } Real getMinLatency() const; Real getLatencyPercentile(Int percent, time_t since = 0) const; ///< Round trip that this share of the samples in the window, from since on, stays within + Int getSampleCountSince(time_t since) const; Real getJitter() const { return m_jitter; } Real getLossPercent() const; ///< Share of retries the earlier send was not acked for Real getLateAckPercent() const; ///< Share of retries whose earlier send was acked after the retry Real getRetryPercent() const; ///< Share of retries, lost and late acked together Real getLatePercent() const; ///< Share of acks that took more than twice the smoothed round trip time. Int getSampleCount() const { return m_samples.size(); } + enum { MAX_REDUNDANCY_DEPTH = 16 }; + Int getRedundancyDepth() const { return m_depth; } + Int getRedundancyCap() const { return m_depthCap; } private: enum @@ -106,6 +111,9 @@ class ConnectionStats Real m_jitter; Real m_previousRtt; Bool m_hasRtt; + + Int m_depth; + Int m_depthCap; }; // Tracks how many frames before its execution frame the frame commands of one sender arrive, diff --git a/Core/GameEngine/Include/GameNetwork/FrameData.h b/Core/GameEngine/Include/GameNetwork/FrameData.h index aa0427967e8..c9fec4e1f11 100644 --- a/Core/GameEngine/Include/GameNetwork/FrameData.h +++ b/Core/GameEngine/Include/GameNetwork/FrameData.h @@ -48,7 +48,7 @@ class FrameData { NetCommandList * getCommandList(); void setFrameCommandCount(UnsignedInt totalCommandCount); UnsignedInt getFrameCommandCount(); - void addCommand(NetCommandMsg *msg); + Bool addCommand(NetCommandMsg *msg); ///< Returns FALSE when the command was already in the frame. UnsignedInt getCommandCount(); void zeroFrame(); void destroyGameMessages(); diff --git a/Core/GameEngine/Include/GameNetwork/FrameDataManager.h b/Core/GameEngine/Include/GameNetwork/FrameDataManager.h index 1ad6055da05..97e98ef86f8 100644 --- a/Core/GameEngine/Include/GameNetwork/FrameDataManager.h +++ b/Core/GameEngine/Include/GameNetwork/FrameDataManager.h @@ -38,7 +38,7 @@ class FrameDataManager : public MemoryPoolObject void reset(); void update(); - void addNetCommandMsg(NetCommandMsg *msg); + Bool addNetCommandMsg(NetCommandMsg *msg); ///< Returns FALSE when the command was already in its frame. void setIsLocal(Bool isLocal); FrameDataReturnType allCommandsReady(UnsignedInt frame, Bool debugSpewage); NetCommandList * getFrameCommandList(UnsignedInt frame); diff --git a/Core/GameEngine/Include/GameNetwork/NetCommandRef.h b/Core/GameEngine/Include/GameNetwork/NetCommandRef.h index 86603e10f3e..2aa842461ce 100644 --- a/Core/GameEngine/Include/GameNetwork/NetCommandRef.h +++ b/Core/GameEngine/Include/GameNetwork/NetCommandRef.h @@ -62,6 +62,13 @@ class NetCommandRef : public MemoryPoolObject time_t getTimeLastSent() const; void setTimeLastSent(time_t timeLastSent); + time_t getTimeFirstSent() const; + void setTimeFirstSent(time_t timeFirstSent); + + Int getRedundantCopies() const; + time_t getTimeFirstRedundant() const; + void noteRedundantCopy(time_t now); ///< One more redundant copy went out, the first one is timestamped. + Bool isRetried() const; void setRetried(Bool retried); @@ -71,7 +78,10 @@ class NetCommandRef : public MemoryPoolObject NetCommandRef *m_prev; UnsignedByte m_relay; ///< Need this in the command reference since the relay value will be different depending on where this particular reference is being sent. time_t m_timeLastSent; - Bool m_retried; ///< Set once this command was sent again, so its last send time is not its first. + Int m_redundantCopies; + time_t m_timeFirstRedundant; + time_t m_timeFirstSent; ///< Stays put on resends, round trips are measured from it + Bool m_retried; ///< Sent again after the redundancy window: no round trip sample #ifdef DEBUG_NETCOMMANDREF UnsignedInt m_id; @@ -142,6 +152,35 @@ inline void NetCommandRef::setTimeLastSent(time_t timeLastSent) m_timeLastSent = timeLastSent; } +inline time_t NetCommandRef::getTimeFirstSent() const +{ + return m_timeFirstSent; +} + +inline void NetCommandRef::setTimeFirstSent(time_t timeFirstSent) +{ + m_timeFirstSent = timeFirstSent; +} + +inline Int NetCommandRef::getRedundantCopies() const +{ + return m_redundantCopies; +} + +inline time_t NetCommandRef::getTimeFirstRedundant() const +{ + return m_timeFirstRedundant; +} + +inline void NetCommandRef::noteRedundantCopy(time_t now) +{ + if (m_redundantCopies == 0) + { + m_timeFirstRedundant = now; + } + ++m_redundantCopies; +} + inline Bool NetCommandRef::isRetried() const { return m_retried; diff --git a/Core/GameEngine/Include/GameNetwork/NetCommandWrapperList.h b/Core/GameEngine/Include/GameNetwork/NetCommandWrapperList.h index 5c6fb8979a6..ad4abfcc772 100644 --- a/Core/GameEngine/Include/GameNetwork/NetCommandWrapperList.h +++ b/Core/GameEngine/Include/GameNetwork/NetCommandWrapperList.h @@ -38,6 +38,7 @@ class NetCommandWrapperListNode : public MemoryPoolObject Bool isComplete(); UnsignedShort getCommandID(); + UnsignedByte getPlayerID(); UnsignedInt getRawDataLength(); void copyChunkData(NetWrapperCommandMsg *msg); UnsignedByte * getRawData(); @@ -48,6 +49,7 @@ class NetCommandWrapperListNode : public MemoryPoolObject protected: UnsignedShort m_commandID; + UnsignedByte m_playerID; UnsignedByte *m_data; UnsignedInt m_totalDataLength; Bool *m_chunksPresent; @@ -72,7 +74,17 @@ class NetCommandWrapperList : public MemoryPoolObject Int getPercentComplete(UnsignedShort wrappedCommandID); protected: + enum { RECENT_COMPLETED_COUNT = 16 }; + void removeFromList(NetCommandWrapperListNode *node); + Bool wasRecentlyCompleted(UnsignedByte playerID, UnsignedShort wrappedCommandID) const; + void noteCompleted(UnsignedByte playerID, UnsignedShort wrappedCommandID); NetCommandWrapperListNode *m_list; + + // A late copy of a chunk must not start a second assembly of a command that was already delivered. + UnsignedByte m_recentCompletedPlayers[RECENT_COMPLETED_COUNT]; + UnsignedShort m_recentCompletedIDs[RECENT_COMPLETED_COUNT]; + Int m_recentCompletedCount; + Int m_recentCompletedNext; }; diff --git a/Core/GameEngine/Include/GameNetwork/NetworkSettings.h b/Core/GameEngine/Include/GameNetwork/NetworkSettings.h index af102d9109d..28f10cecbdd 100644 --- a/Core/GameEngine/Include/GameNetwork/NetworkSettings.h +++ b/Core/GameEngine/Include/GameNetwork/NetworkSettings.h @@ -9,8 +9,16 @@ struct NetworkSettings { struct Send { + std::string mode = "datagram"; // "reliable" = GNS retransmits, "datagram" = unreliable datagrams int frames_per_packet = 2; // send interval in frames + int frames_per_packet_lossy = 1; // send interval in frames while the redundancy depth is above 0, 0 = keep the configured one bool flush_every_frame = true; // flush GNS after each frame + bool redundancy = true; // repeat un-acked commands in packets that go out anyway + int redundancy_window_ms = 0; // wait before a command is repeated, 0 = max(2 x smoothed rtt, 100 ms) + int redundancy_burst_ms = 150; // longest burst of lost packets repaired, caps the depth + bool fast_resend = true; // resend after srtt + 2 x jitter (at most 400 ms) instead of 4 x jitter (at most 1 s) + + bool IsDatagram() const { return mode == "datagram"; } } send; struct InputDelay { diff --git a/Core/GameEngine/Source/GameNetwork/Connection.cpp b/Core/GameEngine/Source/GameNetwork/Connection.cpp index 2831f12c436..6df132bad1e 100644 --- a/Core/GameEngine/Source/GameNetwork/Connection.cpp +++ b/Core/GameEngine/Source/GameNetwork/Connection.cpp @@ -29,6 +29,11 @@ #include "GameNetwork/networkutil.h" #include "GameLogic/GameLogic.h" #include "../NGMP_include.h" +#if defined(GENERALS_ONLINE) +#include "../ngmp_interfaces.h" +#include "GameNetwork/GeneralsOnline/NGMPGame.h" +#include "../NetworkMesh.h" +#endif #if defined(GENERALS_ONLINE) enum { MaxQuitFlushTime = 5000 }; // wait this many milliseconds at most to retry things before quitting @@ -36,6 +41,11 @@ enum { MaxQuitFlushTime = 5000 }; // wait this many milliseconds at most to retr enum { MaxQuitFlushTime = 30000 }; // wait this many milliseconds at most to retry things before quitting #endif +#if defined(GENERALS_ONLINE) +enum { MinScaledRetryTime = 50, MinAutoRedundancyWindow = 100, MaxRedundantCandidates = 128, MinLateAckMargin = 50, MaxScaledRetryTime = 1000, MaxFastRetryTime = 400, StartDepth = 2 }; +enum { DataLossWindowMs = 3000, DataLossBurstRun = 2 }; // the peer's reports of the last 3 s, a lost run of 2 packets is a burst +#endif + /** * The constructor. */ @@ -46,8 +56,15 @@ Connection::Connection() { m_retryTime = 2000; // set retry time to 2 seconds. m_lastTimeSent = 0; m_frameGrouping = 1; + m_configuredFrameGrouping = 1; m_isQuitting = false; m_quitTime = 0; + resetRedundancy(); + m_slot = -1; +#if defined(GENERALS_ONLINE) + m_networkSettings = NetworkSettings::LocalDefaults(); + m_unreliableTransport = FALSE; +#endif } /** @@ -78,7 +95,9 @@ void Connection::init() { m_lastTimeSent = 0; m_frameGrouping = 1; + m_configuredFrameGrouping = 1; m_stats.reset(); + resetRedundancy(); m_isQuitting = FALSE; m_quitTime = 0; } @@ -146,6 +165,7 @@ void Connection::sendNetCommandMsg(NetCommandMsg *msg, UnsignedByte relay) { packet->reset(); NetCommandRef *tempref = NEW_NETCOMMANDREF(msg); + tempref->setRelay(relay); // the relay byte takes room in the packet too Bool msgFits = packet->addCommand(tempref); deleteInstance(tempref); // delete the temporary reference. @@ -240,6 +260,274 @@ void Connection::setQuitting() DEBUG_LOG(("Connection::setQuitting() at time %d", m_quitTime)); } +void Connection::resetRedundancy() { + m_redundancyDepth = 0; + m_redundancyCap = 0; + m_lastLossEventTime = 0; + m_lastDepthChangeTime = 0; + m_queueSuspected = FALSE; +} + +#if defined(GENERALS_ONLINE) +// The mesh connection to the peer in a slot, nullptr when there is none. +static PlayerConnection *GetMeshConnection(Int slot) { + if (TheNGMPGame == nullptr || slot < 0) { + return nullptr; + } + const NGMPGameSlot *gameSlot = (const NGMPGameSlot *)TheNGMPGame->getConstSlot(slot); + if (gameSlot == nullptr) { + return nullptr; + } + NetworkMesh *mesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + return (mesh != nullptr) ? mesh->GetConnectionForUser(gameSlot->m_userID) : nullptr; +} +#endif + +// While the redundancy depth is above 0 the connection sends at the lossy frame grouping, otherwise at the configured one. +void Connection::applyFrameGrouping() { + time_t grouping = m_configuredFrameGrouping; +#if defined(GENERALS_ONLINE) + if (m_redundancyDepth > 0) { + const Int lossyFrames = m_networkSettings.send.frames_per_packet_lossy; + if (lossyFrames > 0) { + const time_t lossy = (time_t)(lossyFrames * 1000 / GENERALS_ONLINE_HIGH_FPS_LIMIT); + if (lossy < grouping) { + grouping = lossy; + } + } + } +#endif + m_frameGrouping = grouping; +} + +void Connection::setRedundancyDepth(Int depth, Int cap, time_t now) { +#if defined(GENERALS_ONLINE) + if (depth != m_redundancyDepth) { + m_lastDepthChangeTime = now; + } + m_redundancyDepth = depth; + m_stats.setRedundancyDepth(depth, cap); + applyFrameGrouping(); +#endif +} + +// How our packets fared on their way to the peer, from the peer's own reports; FALSE when there are none, then nothing is gated on them. +Bool Connection::getOutgoingLoss(float &lossPercent, Int &longestRun) { +#if defined(GENERALS_ONLINE) + PlayerConnection *connection = GetMeshConnection(m_slot); + PacketFeedback::Summary summary; + if (connection == nullptr || !connection->GetOutgoingLoss(DataLossWindowMs, summary)) { + return FALSE; + } + lossPercent = summary.lossPercent; + longestRun = summary.longestRun; + return TRUE; +#else + return FALSE; +#endif +} + +// A resend or a late ack on a link that loses nothing is delay, and repeating commands cannot help it. +// TRUE when there is no packet feedback to say otherwise. +Bool Connection::hasLossEvidence() { +#if defined(GENERALS_ONLINE) + PlayerConnection *connection = GetMeshConnection(m_slot); + PacketFeedback::Summary outLoss; + PacketFeedback::Summary inLoss; + if (connection == nullptr) { + return TRUE; + } + const Bool hasOut = connection->GetOutgoingLoss(NETWORK_STATS_WINDOW_MS, outLoss); + const Bool hasIn = connection->GetIncomingLoss(NETWORK_STATS_WINDOW_MS, inLoss); + if (!hasOut && !hasIn) { + return TRUE; + } + return (hasOut && (outLoss.lossPercent > 0.0f || outLoss.longestRun > 0)) || (hasIn && (inLoss.lossPercent > 0.0f || inLoss.longestRun > 0)); +#else + return TRUE; +#endif +} + +// A loss takes the depth to 2, a burst (all copies of a command lost) one deeper. +void Connection::noteLossEvent(time_t now, Bool burst) { +#if defined(GENERALS_ONLINE) + if (!m_networkSettings.send.redundancy) { + return; + } + + if (!hasLossEvidence()) { + return; + } + + // a full queue loses packets because it is full, more copies would only fill it further + if (m_queueSuspected) { + return; + } + + m_lastLossEventTime = now; + + Int target = m_redundancyDepth; + if (m_redundancyDepth < StartDepth) { + target = StartDepth; + } else if (burst) { + // retries of commands sent before the last change say nothing about the new depth, one round trip has to pass + time_t settle = (time_t)(2.0f * m_stats.getLatency()); + if (settle < MinAutoRedundancyWindow) { + settle = MinAutoRedundancyWindow; + } + if ((now - m_lastDepthChangeTime) >= settle) { + // a lost ack also leaves copies unacked: only a lost run of 2+ packets (peer report) counts as a burst + float outLoss = 0.0f; + Int outRun = 0; + if (!getOutgoingLoss(outLoss, outRun) || outRun >= DataLossBurstRun) { + target = m_redundancyDepth + 1; + } + } + } + + if (target > m_redundancyCap) { + target = m_redundancyCap; + } + if (target != m_redundancyDepth) { + setRedundancyDepth(target, m_redundancyCap, now); + } +#endif +} + +Bool Connection::isQueueBuilding(time_t now) { +#if defined(GENERALS_ONLINE) + enum { RECENT_MS = 3000, MIN_SAMPLES = 20, MIN_RISE_MS = 50 }; + const time_t since = now - RECENT_MS; + if (m_stats.getSampleCountSince(since) < MIN_SAMPLES) { + return FALSE; + } + const Real low = m_stats.getMinLatency(); + const Real recent = m_stats.getLatencyPercentile(50, since); + if (low <= 0.0f || recent < 2.0f * low || (recent - low) < (Real)MIN_RISE_MS) { + return FALSE; + } + float outLoss = 0.0f; + Int outRun = 0; + const Bool outLossing = getOutgoingLoss(outLoss, outRun) && outLoss >= 1.0f; + return outLossing || m_stats.getRetryPercent() >= 2.0f; +#else + return FALSE; +#endif +} + +void Connection::updateRedundancy(time_t curtime, Bool &active, time_t &window, Int &depth) { + active = FALSE; + window = 0; + depth = 0; + m_retryTime = 2000; + +#if defined(GENERALS_ONLINE) + const NetworkSettings::Send &sendSettings = m_networkSettings.send; + const Real srtt = m_stats.getLatency(); + const time_t interval = (m_frameGrouping > 1) ? m_frameGrouping : 1; + + if ((sendSettings.redundancy || m_unreliableTransport) && (srtt > 0.0f)) { + const Bool fastResend = sendSettings.fast_resend; + // the peer sends its acks with its next packet, up to one send interval after it got the command + time_t retry = (time_t)(srtt + (fastResend ? 2.0f : 4.0f) * m_stats.getJitter()) + interval; + if (retry < MinScaledRetryTime) { + retry = MinScaledRetryTime; + } + if (retry > (fastResend ? MaxFastRetryTime : MaxScaledRetryTime)) { + retry = fastResend ? MaxFastRetryTime : MaxScaledRetryTime; + } + m_retryTime = retry; + } + + if (!sendSettings.redundancy) { + if (m_redundancyDepth != 0) { + setRedundancyDepth(0, 0, curtime); + } + return; + } + + // the deepest depth covers the longest burst for the current send interval + Int cap = (Int)((sendSettings.redundancy_burst_ms + interval - 1) / interval); + if (cap > ConnectionStats::MAX_REDUNDANCY_DEPTH) { + cap = ConnectionStats::MAX_REDUNDANCY_DEPTH; + } + if (cap < 1) { + cap = 1; + } + + if (cap != m_redundancyCap) { + m_redundancyCap = cap; + if (m_redundancyDepth > cap) { + setRedundancyDepth(cap, cap, curtime); + } + } + + const Bool queue = isQueueBuilding(curtime); + m_queueSuspected = queue; + if (queue && m_redundancyDepth > 0) { + // one step per settle time, the next round trips show whether the queue drains + time_t settle = (time_t)(2.0f * srtt); + if (settle < MinAutoRedundancyWindow) { + settle = MinAutoRedundancyWindow; + } + if ((curtime - m_lastDepthChangeTime) >= settle) { + setRedundancyDepth(m_redundancyDepth - 1, cap, curtime); + } + } + + if ((m_redundancyDepth > 0) && ((curtime - m_lastLossEventTime) >= NETWORK_STATS_WINDOW_MS)) { + setRedundancyDepth(m_redundancyDepth - 1, cap, curtime); + m_lastLossEventTime = curtime; + } + m_stats.setRedundancyDepth(m_redundancyDepth, cap); + + depth = m_redundancyDepth; + active = (depth > 0); + + window = sendSettings.redundancy_window_ms; + if (window <= 0) { + window = (time_t)(2.0f * srtt); + if (window < MinAutoRedundancyWindow) { + window = MinAutoRedundancyWindow; + } + } +#endif +} + +// Fills the rest of the packet with young un-acked commands, so the next packet repairs a loss without a retry. +void Connection::addRedundantCommands(NetPacket *packet, time_t curtime, time_t window, Int depth) { +#if defined(GENERALS_ONLINE) + NetCommandRef *candidates[MaxRedundantCandidates]; + Int count = 0; + const time_t interval = (m_frameGrouping > 1) ? m_frameGrouping : 1; // a command sent in this packet is younger than this + + NetCommandRef *ref = m_netCommandList->getFirstMessage(); + while ((ref != nullptr) && (count < MaxRedundantCandidates)) { + const time_t timeLastSent = ref->getTimeLastSent(); + if ((timeLastSent != -1) && CommandRequiresAck(ref->getCommand()) + && ((curtime - timeLastSent) >= interval) && ((curtime - ref->getTimeFirstSent()) <= window) + && (ref->getRedundantCopies() < depth)) { + Int slot = count; + while ((slot > 0) && (candidates[slot - 1]->getTimeFirstSent() > ref->getTimeFirstSent())) { + candidates[slot] = candidates[slot - 1]; + --slot; + } + candidates[slot] = ref; + ++count; + } + ref = ref->getNext(); + } + + for (Int i = 0; i < count; ++i) { + if (!packet->addCommand(candidates[i])) { + break; + } + candidates[i]->noteRedundantCopy(curtime); + candidates[i]->setTimeLastSent(curtime); // not a retry, so the round trip is still taken from the first send + } +#endif +} + /** * This is the good part. We take all the network commands queued up for this connection, * packetize them and put them on the transport's send queue for actual sending. @@ -265,6 +553,16 @@ UnsignedInt Connection::doSend() { return 0; } + Bool redundancy = FALSE; + time_t redundancyWindow = 0; + Int redundancyDepth = 0; + updateRedundancy(curtime, redundancy, redundancyWindow, redundancyDepth); +#if defined(GENERALS_ONLINE) + const Bool fastResend = m_networkSettings.send.fast_resend; +#else + const Bool fastResend = FALSE; +#endif + // iterate through all the messages and put them into a packet(s). NetCommandRef *msg = m_netCommandList->getFirstMessage(); @@ -281,25 +579,46 @@ UnsignedInt Connection::doSend() { time_t timeLastSent = msg->getTimeLastSent(); - if (((curtime - timeLastSent) > m_retryTime) || (timeLastSent == -1)) { - notDone = packet->addCommand(msg); - if (notDone) { + // a command inside the redundancy window is not retried, it goes out again as a redundant copy + // fast resend: the first resend counts from the first send, not from the last redundant copy, and the window does not hold it back + const time_t sentAt = (fastResend && !msg->isRetried() && (timeLastSent != -1)) ? msg->getTimeFirstSent() : timeLastSent; + Bool due = (timeLastSent == -1) || ((curtime - sentAt) > m_retryTime); + if (due && redundancy && !fastResend && (timeLastSent != -1) && ((curtime - msg->getTimeFirstSent()) <= redundancyWindow)) { + due = FALSE; + } + + if (due) { + if (packet->addCommand(msg)) { // the msg command was added to the packet. if (CommandRequiresAck(msg->getCommand())) { if (timeLastSent != -1) { m_stats.noteRetry(curtime); + if (!msg->isRetried() && !fastResend) { + noteLossEvent(curtime, TRUE); // the command and all its copies were lost, or their acks were + } msg->setRetried(TRUE); + } else { + msg->setTimeFirstSent(curtime); } msg->setTimeLastSent(curtime); } else { m_netCommandList->removeMessage(msg); deleteInstance(msg); } + } else { + notDone = FALSE; + if (packet->getNumCommands() > 0) { + next = msg; // the packet is full, this one starts the next packet + } } } msg = next; } + if (redundancy && notDone && (packet->getNumCommands() > 0)) { + addRedundantCommands(packet, curtime, redundancyWindow, redundancyDepth); + } + if (msg != nullptr) { DEBUG_LOG(("didn't finish sending all commands in connection")); } @@ -367,14 +686,37 @@ NetCommandRef * Connection::processAck(UnsignedShort commandID, UnsignedByte ori #endif // A retried command was last sent after its first send, so its round trip time would be too short. + // Redundant copies are not retries: any copy's ack counts as an ack of the first send. const time_t now = timeGetTime(); if (temp->isRetried()) { - // an ack sooner after the retry than any round trip so far answers the earlier send: the retry went out for a late ack, not a loss + // an ack faster than any round trip so far answers the earlier send: the retry was for a late ack +#if defined(GENERALS_ONLINE) + const Bool fastResend = m_networkSettings.send.fast_resend; +#else + const Bool fastResend = FALSE; +#endif if ((m_stats.getSampleCount() > 0) && ((now - temp->getTimeLastSent()) < (time_t)(0.9f * m_stats.getMinLatency()))) { m_stats.noteLateAck(now); + if (fastResend) { + // unambiguous: it answers the first send, the slow acks that cause fast resends must not be missing from srtt + m_stats.noteAcked((Real)(now - temp->getTimeFirstSent()), now); + } + } else if (fastResend) { + // Karn: no sample from an ambiguous ack; the burst event moves here from the resend, a slow ack is no loss + noteLossEvent(now, TRUE); } } else { - m_stats.noteAcked((Real)(now - temp->getTimeLastSent()), now); + const Real rtt = (Real)(now - temp->getTimeFirstSent()); + const Real srtt = m_stats.getLatency(); + const Real lateMargin = (srtt > MinLateAckMargin) ? srtt : (Real)MinLateAckMargin; + if ((m_stats.getSampleCount() > 0) && ((rtt - srtt) > lateMargin)) { + noteLossEvent(now, FALSE); // an ack later than the smoothed round trip plus its own length, at least 50 ms + } else if ((temp->getTimeFirstRedundant() != -1) && (m_stats.getSampleCount() > 0) + && ((now - temp->getTimeFirstRedundant()) >= (time_t)m_stats.getMinLatency() - m_frameGrouping / 2)) { + // the ack came too late to answer the first copy, a repeat repaired a loss + noteLossEvent(now, FALSE); + } + m_stats.noteAcked((Real)(now - temp->getTimeFirstSent()), now); } #if defined(RTS_DEBUG) @@ -387,7 +729,8 @@ NetCommandRef * Connection::processAck(UnsignedShort commandID, UnsignedByte ori } void Connection::setFrameGrouping(time_t frameGrouping) { - m_frameGrouping = frameGrouping; + m_configuredFrameGrouping = frameGrouping; + applyFrameGrouping(); // m_retryTime = frameGrouping * 4; } diff --git a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp index abe73435b7a..9d459b074a4 100644 --- a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp +++ b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp @@ -360,6 +360,9 @@ void ConnectionManager::init() m_netCommandWrapperList = newInstance(NetCommandWrapperList); m_netCommandWrapperList->init(); +#if defined(GENERALS_ONLINE) + resetRecentCommandIds(); +#endif s_fileCommandMap.clear(); s_fileRecipientMaskMap.clear(); @@ -410,6 +413,9 @@ void ConnectionManager::reset() m_netCommandWrapperList->init(); } m_netCommandWrapperList->reset(); +#if defined(GENERALS_ONLINE) + resetRecentCommandIds(); +#endif m_localSlot = -1; #ifdef MEMORYPOOL_DEBUG @@ -484,6 +490,60 @@ void ConnectionManager::destroyGameMessages() { } } +#if defined(GENERALS_ONLINE) +// TRUE if (sender, id) was seen in the last 256 ids, remembers it otherwise. Synchronized commands are deduplicated by the frame data. +Bool ConnectionManager::isDuplicateCommand(NetCommandMsg *msg) { + if (IsCommandSynchronized(msg->getNetCommandType())) { + return FALSE; + } + + const UnsignedInt playerID = msg->getPlayerID(); + if (playerID >= MAX_SLOTS) { + return FALSE; + } + + RecentCommandIds &recent = m_recentIds[playerID]; + const UnsignedShort id = msg->getID(); + + if (!recent.valid) { + recent.valid = TRUE; + recent.highest = id; + for (Int i = 0; i < RECENT_ID_COUNT; ++i) { + recent.seen[i] = FALSE; + } + recent.seen[id % RECENT_ID_COUNT] = TRUE; + return FALSE; + } + + const Int diff = (Int)(Short)(id - recent.highest); + if (diff > 0) { + // the window moves forward, the slots it moves over belong to ids not seen yet + for (Int step = 1; step <= diff && step <= RECENT_ID_COUNT; ++step) { + recent.seen[(recent.highest + step) % RECENT_ID_COUNT] = FALSE; + } + recent.highest = id; + recent.seen[id % RECENT_ID_COUNT] = TRUE; + return FALSE; + } + + if (-diff >= RECENT_ID_COUNT) { + return FALSE; + } + + if (recent.seen[id % RECENT_ID_COUNT]) { + return TRUE; + } + recent.seen[id % RECENT_ID_COUNT] = TRUE; + return FALSE; +} + +void ConnectionManager::resetRecentCommandIds() { + for (Int i = 0; i < MAX_SLOTS; ++i) { + m_recentIds[i].valid = FALSE; + } +} +#endif + /** * ConnectionManager::doRelay() * Queries the transport for commands that need to be relayed to another client. @@ -513,6 +573,11 @@ void ConnectionManager::doRelay() { if (CommandRequiresAck(cmd->getCommand())) { ackCommand(cmd, m_localSlot); +#if defined(GENERALS_ONLINE) + if (isDuplicateCommand(cmd->getCommand())) { + continue; // a redundant copy, acked again in case the first ack was lost + } +#endif } if (!processNetCommand(cmd)) { sendRemoteCommand(cmd); @@ -537,6 +602,11 @@ void ConnectionManager::doRelay() { for (NetCommandRef* cmd = cmdList->getFirstMessage(); cmd; cmd = cmd->getNext()) { if (CommandRequiresAck(cmd->getCommand())) { ackCommand(cmd, m_localSlot); +#if defined(GENERALS_ONLINE) + if (isDuplicateCommand(cmd->getCommand())) { + continue; + } +#endif } if (!processNetCommand(cmd)) { sendRemoteCommand(cmd); @@ -1008,9 +1078,14 @@ void ConnectionManager::processFrameInfo(NetFrameCommandMsg *msg) { if (playerID < MAX_SLOTS) { if (m_frameData[playerID] != nullptr) { // DEBUG_LOG(("ConnectionManager::processFrameInfo - player %d, frame %d, command count %d, received on frame %d", playerID, msg->getExecutionFrame(), msg->getCommandCount(), TheGameLogic->getFrame())); +#if defined(GENERALS_ONLINE) + // a redundant copy of a frame that was already seen says nothing about how early frames arrive + NetCommandList *frameCommands = m_frameData[playerID]->getFrameCommandList(msg->getExecutionFrame()); + const Bool isCopy = (frameCommands != nullptr) && (frameCommands->findMessage(msg) != nullptr); +#endif m_frameData[playerID]->setFrameCommandCount(msg->getExecutionFrame(), msg->getCommandCount()); #if defined(GENERALS_ONLINE) - if ((playerID != (UnsignedInt)m_localSlot) && (TheGameLogic != nullptr)) { + if (!isCopy && (playerID != (UnsignedInt)m_localSlot) && (TheGameLogic != nullptr)) { m_cushionStats[playerID].note((Int)msg->getExecutionFrame() - (Int)TheGameLogic->getFrame(), timeGetTime()); } #endif @@ -1311,13 +1386,20 @@ void ConnectionManager::sendRemoteCommand(NetCommandRef *msg) { const UnsignedByte relay = msg->getRelay(); const UnsignedInt playerID = msg->getCommand()->getPlayerID(); FrameDataManager *frameDataMgr = playerID < MAX_SLOTS ? m_frameData[playerID] : nullptr; + Bool isNew = TRUE; if ((relay & (1 << m_localSlot)) && (frameDataMgr != nullptr)) { if (IsCommandSynchronized(msg->getCommand()->getNetCommandType())) { DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("ConnectionManager::sendRemoteCommand - adding net command of type %s to player %d for frame %d", GetNetCommandTypeAsString(msg->getCommand()->getNetCommandType()), msg->getCommand()->getPlayerID(), msg->getCommand()->getExecutionFrame())); - frameDataMgr->addNetCommandMsg(msg->getCommand()); + isNew = frameDataMgr->addNetCommandMsg(msg->getCommand()); } } +#if defined(GENERALS_ONLINE) + if (!isNew) { + return; // a redundant copy, it was already relayed and counted + } +#endif + for (Int i = 0; i < MAX_SLOTS; ++i) { if ((relay & (1 << i)) && ((m_connections[i] != nullptr) && (m_connections[i]->isQuitting() == FALSE))) { DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("ConnectionManager::sendRemoteCommand - relaying command %d to player %d", msg->getCommand()->getID(), i)); @@ -1335,7 +1417,7 @@ void ConnectionManager::sendRemoteCommand(NetCommandRef *msg) { } // Do some metrics to find the minimum packet arrival cushion. - if (IsCommandSynchronized(msg->getCommand()->getNetCommandType())) { + if (isNew && IsCommandSynchronized(msg->getCommand()->getNetCommandType())) { // DEBUG_LOG(("ConnectionManager::sendRemoteCommand - about to call allCommandsReady")); if (allCommandsReady(msg->getCommand()->getExecutionFrame(), TRUE)) { UnsignedInt cushion = msg->getCommand()->getExecutionFrame() - TheGameLogic->getFrame(); @@ -2323,6 +2405,10 @@ void ConnectionManager::parseUserList(const GameInfo *game) { m_connections[i] = newInstance(Connection)(); m_connections[i]->init(); +#if defined(GENERALS_ONLINE) + m_connections[i]->setNetworkSettings(m_networkSettings, TheLAN != nullptr || m_networkSettings.send.IsDatagram()); +#endif + m_connections[i]->setSlot(i); m_connections[i]->attachTransport(m_transport); // UnsignedShort port = (TheNAT)?TheNAT->getSlotPort(i):8088; UnsignedShort port = slot->getPort(); diff --git a/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp b/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp index 08db5856949..129659295a2 100644 --- a/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp +++ b/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp @@ -37,6 +37,8 @@ void ConnectionStats::reset() m_jitter = 0.0f; m_previousRtt = 0.0f; m_hasRtt = FALSE; + m_depth = 0; + m_depthCap = 0; } void ConnectionStats::noteAcked(Real rttMs, time_t now) @@ -86,6 +88,21 @@ void ConnectionStats::noteLateAck(time_t now) m_lateAcks.push(stamp); } +void ConnectionStats::setRedundancyDepth(Int depth, Int cap) +{ + if (depth < 0) + { + depth = 0; + } + if (depth > MAX_REDUNDANCY_DEPTH) + { + depth = MAX_REDUNDANCY_DEPTH; + } + + m_depth = depth; + m_depthCap = cap; +} + void ConnectionStats::update(time_t now) { m_samples.prune(now, NETWORK_STATS_WINDOW_MS); @@ -141,6 +158,20 @@ Real ConnectionStats::getLatencyPercentile(Int percent, time_t since) const return sorted[index]; } +Int ConnectionStats::getSampleCountSince(time_t since) const +{ + Int count = 0; + for (Int i = m_samples.size() - 1; i >= 0; --i) + { + if (m_samples[i].time < since) + { + break; + } + ++count; + } + return count; +} + Real ConnectionStats::getRetryPercent() const { const Int total = m_samples.size() + m_retries.size(); diff --git a/Core/GameEngine/Source/GameNetwork/FrameData.cpp b/Core/GameEngine/Source/GameNetwork/FrameData.cpp index 88b980f33f5..3c102507bf1 100644 --- a/Core/GameEngine/Source/GameNetwork/FrameData.cpp +++ b/Core/GameEngine/Source/GameNetwork/FrameData.cpp @@ -155,7 +155,7 @@ UnsignedInt FrameData::getCommandCount() { /** * Add a command to this frame */ -void FrameData::addCommand(NetCommandMsg *msg) { +Bool FrameData::addCommand(NetCommandMsg *msg) { // need to add the message in order of command ID if (m_commandList == nullptr) { init(); @@ -164,12 +164,13 @@ void FrameData::addCommand(NetCommandMsg *msg) { // We don't need to worry about setting the relay since its not getting sent anywhere. if (m_commandList->findMessage(msg) != nullptr) { // We don't want to add the same command twice. - return; + return FALSE; } m_commandList->addMessage(msg); ++m_commandCount; //DEBUG_LOG(("added command %d, type = %d(%s), command count = %d, frame command count = %d", msg->getID(), msg->getNetCommandType(), GetNetCommandTypeAsString(msg->getNetCommandType()), m_commandCount, m_frameCommandCount)); + return TRUE; } /** diff --git a/Core/GameEngine/Source/GameNetwork/FrameDataManager.cpp b/Core/GameEngine/Source/GameNetwork/FrameDataManager.cpp index 1dfc3349306..871275ae4e2 100644 --- a/Core/GameEngine/Source/GameNetwork/FrameDataManager.cpp +++ b/Core/GameEngine/Source/GameNetwork/FrameDataManager.cpp @@ -83,16 +83,17 @@ void FrameDataManager::update() { /** * Add a network command to the appropriate frame. */ -void FrameDataManager::addNetCommandMsg(NetCommandMsg *msg) { +Bool FrameDataManager::addNetCommandMsg(NetCommandMsg *msg) { UnsignedInt frame = msg->getExecutionFrame(); UnsignedInt frameindex = frame % FRAME_DATA_LENGTH; DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("FrameDataManager::addNetCommandMsg - about to add a command of type %s for frame %d, frame index %d", GetNetCommandTypeAsString(msg->getNetCommandType()), frame, frameindex)); - m_frameData[frameindex].addCommand(msg); + const Bool added = m_frameData[frameindex].addCommand(msg); if (m_isLocal) { // If this is the local connection, adjust the frame command count. m_frameData[frameindex].setFrameCommandCount(m_frameData[frameindex].getCommandCount()); } + return added; } /** diff --git a/Core/GameEngine/Source/GameNetwork/NetCommandRef.cpp b/Core/GameEngine/Source/GameNetwork/NetCommandRef.cpp index 3c470673176..6bb92c16b1f 100644 --- a/Core/GameEngine/Source/GameNetwork/NetCommandRef.cpp +++ b/Core/GameEngine/Source/GameNetwork/NetCommandRef.cpp @@ -49,6 +49,9 @@ NetCommandRef::NetCommandRef(NetCommandMsg *msg) m_prev = nullptr; m_relay = 0; m_timeLastSent = -1; + m_timeFirstSent = -1; + m_redundantCopies = 0; + m_timeFirstRedundant = -1; m_retried = FALSE; #ifdef DEBUG_NETCOMMANDREF diff --git a/Core/GameEngine/Source/GameNetwork/NetCommandWrapperList.cpp b/Core/GameEngine/Source/GameNetwork/NetCommandWrapperList.cpp index 1e50c5f53e5..79aa405544d 100644 --- a/Core/GameEngine/Source/GameNetwork/NetCommandWrapperList.cpp +++ b/Core/GameEngine/Source/GameNetwork/NetCommandWrapperList.cpp @@ -49,6 +49,7 @@ NetCommandWrapperListNode::NetCommandWrapperListNode(NetWrapperCommandMsg *msg) m_data = NEW UnsignedByte[m_totalDataLength]; // pool[]ify m_commandID = msg->getWrappedCommandID(); + m_playerID = msg->getPlayerID(); } NetCommandWrapperListNode::~NetCommandWrapperListNode() { @@ -74,6 +75,10 @@ UnsignedShort NetCommandWrapperListNode::getCommandID() { return m_commandID; } +UnsignedByte NetCommandWrapperListNode::getPlayerID() { + return m_playerID; +} + UnsignedInt NetCommandWrapperListNode::getRawDataLength() { return m_totalDataLength; } @@ -133,6 +138,8 @@ UnsignedByte * NetCommandWrapperListNode::getRawData() { NetCommandWrapperList::NetCommandWrapperList() { m_list = nullptr; + m_recentCompletedCount = 0; + m_recentCompletedNext = 0; } NetCommandWrapperList::~NetCommandWrapperList() { @@ -146,6 +153,8 @@ NetCommandWrapperList::~NetCommandWrapperList() { void NetCommandWrapperList::init() { m_list = nullptr; + m_recentCompletedCount = 0; + m_recentCompletedNext = 0; } void NetCommandWrapperList::reset() { @@ -155,6 +164,26 @@ void NetCommandWrapperList::reset() { deleteInstance(m_list); m_list = temp; } + m_recentCompletedCount = 0; + m_recentCompletedNext = 0; +} + +Bool NetCommandWrapperList::wasRecentlyCompleted(UnsignedByte playerID, UnsignedShort wrappedCommandID) const { + for (Int i = 0; i < m_recentCompletedCount; ++i) { + if ((m_recentCompletedPlayers[i] == playerID) && (m_recentCompletedIDs[i] == wrappedCommandID)) { + return TRUE; + } + } + return FALSE; +} + +void NetCommandWrapperList::noteCompleted(UnsignedByte playerID, UnsignedShort wrappedCommandID) { + m_recentCompletedPlayers[m_recentCompletedNext] = playerID; + m_recentCompletedIDs[m_recentCompletedNext] = wrappedCommandID; + m_recentCompletedNext = (m_recentCompletedNext + 1) % RECENT_COMPLETED_COUNT; + if (m_recentCompletedCount < RECENT_COMPLETED_COUNT) { + ++m_recentCompletedCount; + } } Int NetCommandWrapperList::getPercentComplete(UnsignedShort wrappedCommandID) @@ -175,6 +204,10 @@ void NetCommandWrapperList::processWrapper(NetCommandRef *ref) { NetCommandWrapperListNode *temp = m_list; NetWrapperCommandMsg *msg = (NetWrapperCommandMsg *)(ref->getCommand()); + if (wasRecentlyCompleted(msg->getPlayerID(), msg->getWrappedCommandID())) { + return; // a duplicate or late chunk of a command that was already delivered + } + while ((temp != nullptr) && (temp->getCommandID() != msg->getWrappedCommandID())) { temp = temp->m_next; } @@ -206,6 +239,7 @@ NetCommandList * NetCommandWrapperList::getReadyCommands() deleteInstance(msg); msg = nullptr; + noteCompleted(temp->getPlayerID(), temp->getCommandID()); removeFromList(temp); temp = nullptr; } diff --git a/Core/GameEngine/Source/GameNetwork/NetPacket.cpp b/Core/GameEngine/Source/GameNetwork/NetPacket.cpp index c04446d7e51..3a76de3b860 100644 --- a/Core/GameEngine/Source/GameNetwork/NetPacket.cpp +++ b/Core/GameEngine/Source/GameNetwork/NetPacket.cpp @@ -437,6 +437,8 @@ NetCommandList * NetPacket::getCommandList() { DEBUG_CRASH(("Unrecognized packet entry, ignoring.")); DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("NetPacket::getCommandList - Unrecognized packet entry at index %d", i)); dumpPacketToLog(m_packet, m_packetLen); + deleteInstance(lastCommand); + lastCommand = nullptr; continue; } @@ -462,6 +464,7 @@ NetCommandList * NetPacket::getCommandList() { // Repeat the last command, doing some funky cool byte-saving stuff if (lastCommand == nullptr) { DEBUG_CRASH(("Got a repeat command with no command to repeat.")); + continue; } NetCommandMsg *msg = nullptr; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index 87c12c5af1c..51205b64f77 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -421,8 +421,14 @@ inline void ReadServiceConfig(const nlohmann::json& j, ServiceConfig& c) if (j.contains("send")) { const nlohmann::json& s = j["send"]; + ReadServiceConfigValue(s, "mode", c.send.mode); ReadServiceConfigValue(s, "frames_per_packet", c.send.frames_per_packet); + ReadServiceConfigValue(s, "frames_per_packet_lossy", c.send.frames_per_packet_lossy); ReadServiceConfigValue(s, "flush_every_frame", c.send.flush_every_frame); + ReadServiceConfigValue(s, "redundancy", c.send.redundancy); + ReadServiceConfigValue(s, "redundancy_window_ms", c.send.redundancy_window_ms); + ReadServiceConfigValue(s, "redundancy_burst_ms", c.send.redundancy_burst_ms); + ReadServiceConfigValue(s, "fast_resend", c.send.fast_resend); } if (j.contains("input_delay")) { @@ -459,6 +465,9 @@ inline void ClampServiceConfig(ServiceConfig& c) c.connection.latency_history_ms = (std::max)(c.connection.latency_history_ms, 1000); c.connection.probe_lobby_interval_ms = std::clamp(c.connection.probe_lobby_interval_ms, 50, 5000); c.connection.probe_match_interval_ms = std::clamp(c.connection.probe_match_interval_ms, 50, 10000); + c.send.redundancy_window_ms = std::clamp(c.send.redundancy_window_ms, 0, 5000); + c.send.redundancy_burst_ms = std::clamp(c.send.redundancy_burst_ms, 0, 1000); + c.send.frames_per_packet_lossy = std::clamp(c.send.frames_per_packet_lossy, 0, 8); c.input_delay.min_frames = std::clamp(c.input_delay.min_frames, 1, 64); c.input_delay.slack_percent = std::clamp(c.input_delay.slack_percent, 0, 200); diff --git a/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp b/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp index 68954303707..ea6a22d989b 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp @@ -7185,7 +7185,7 @@ namespace enum OverlayColumn { COL_SLOT, COL_NAME, COL_BARS, COL_PATH, COL_PROBE, COL_CMD, COL_JIT, COL_LOST, COL_LATE, - COL_CUSHION, COL_SAMPLES, COL_STALLS, COL_OUT, COL_IN, + COL_CUSHION, COL_SAMPLES, COL_STALLS, COL_DEPTH, COL_OUT, COL_IN, COL_COUNT }; @@ -7198,7 +7198,7 @@ namespace const wchar_t *const s_overlayHeaders[COL_COUNT] = { L"Slot", L"Player", L"Bars", L"Path", L"Probe", L"Cmd", L"Jit", L"Lost%", L"Late%", - L"Cush5/50", L"n", L"Stalls", L"OutL%", L"InL%" + L"Cush5/50", L"n", L"Stalls", L"Depth", L"OutL%", L"InL%" }; @@ -7498,6 +7498,8 @@ void InGameUI::drawNetworkOverlay(Int mode) sev[COL_LOST] = overlayPercentSeverity(pCmdStats->getLossPercent()); cell[COL_LATE].format(L"%.0f", pCmdStats->getLatePercent()); sev[COL_LATE] = overlayPercentSeverity(pCmdStats->getLatePercent()); + cell[COL_DEPTH].format(L"%d/%d", pCmdStats->getRedundancyDepth(), pCmdStats->getRedundancyCap()); + sev[COL_DEPTH] = (pCmdStats->getRedundancyCap() > 0 && pCmdStats->getRedundancyDepth() >= pCmdStats->getRedundancyCap()) ? ConnectionRating::SEVERITY_WARN : ConnectionRating::SEVERITY_OK; } else { @@ -7505,6 +7507,7 @@ void InGameUI::drawNetworkOverlay(Int mode) cell[COL_JIT] = L"-"; cell[COL_LOST] = L"-"; cell[COL_LATE] = L"-"; + cell[COL_DEPTH] = L"-"; } if (row.info.cushionSamples > 0) diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp index d1890a35bb7..b6fca95e0f9 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp @@ -1475,13 +1475,13 @@ int PlayerConnection::SendGamePacket(void* pBuffer, uint32_t totalDataSize) if (totalDataSize == 0) { NetworkLog(ELogVerbosity::LOG_RELEASE, "[GAME PACKET] Cannot send empty game packet to user %lld", m_userID); - return (int)k_EResultFail; + return -(int)k_EResultFail; } if (pBuffer == nullptr) { NetworkLog(ELogVerbosity::LOG_RELEASE, "[GAME PACKET] Cannot send game packet with null buffer to user %lld", m_userID); - return (int)k_EResultFail; + return -(int)k_EResultFail; } if (AnticheatPlugInterface::DoesACPluginProvideSecureGameTransport()) @@ -1494,21 +1494,28 @@ int PlayerConnection::SendGamePacket(void* pBuffer, uint32_t totalDataSize) if (m_hSteamConnection == k_HSteamNetConnection_Invalid) { NetworkLog(ELogVerbosity::LOG_RELEASE, "[GAME PACKET] Cannot send game packet - connection is invalid for user %lld", m_userID); - return (int)k_EResultFail; + return -(int)k_EResultFail; } + ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); + + // datagram mode leaves loss repair to the game layer; the restart flag only applies to ISteamNetworkingMessages int sendFlags = k_nSteamNetworkingSend_Reliable | k_nSteamNetworkingSend_AutoRestartBrokenSession; // default from last patch + if (serviceConf.send.IsDatagram()) + { + sendFlags = k_nSteamNetworkingSend_UnreliableNoNagle; + } NetworkLog(ELogVerbosity::LOG_DEBUG, "[GAME PACKET] Sending msg of size %ld to user %lld\n", totalDataSize, m_userID); const int r = SendEnveloped(ENetworkChannel::NETWORK_CHANNEL_GAME, static_cast(pBuffer), totalDataSize, sendFlags); if (r != k_EResultOK) { NetworkLog(ELogVerbosity::LOG_RELEASE, "[GAME PACKET] Failed to send, err code was %d", r); - return r; + return -r; } } - return (int)k_EResultFail; + return 0; } From 7110ab38ed24305a5788d3a05a3a92a3dd23b54e Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Thu, 22 Oct 2026 07:03:26 +0200 Subject: [PATCH 07/12] feat(network): Validate peer identity and authenticate match startup and forwarded traffic --- .../Include/GameNetwork/ConnectionManager.h | 77 ++ .../Include/GameNetwork/NetCommandMsg.h | 56 ++ .../Include/GameNetwork/NetPacketStructs.h | 88 +++ .../Include/GameNetwork/NetworkDefs.h | 7 + .../Include/GameNetwork/NetworkInterface.h | 5 + .../GameLogic/System/GameLogicDispatch.cpp | 15 + .../Source/GameNetwork/Connection.cpp | 10 + .../Source/GameNetwork/ConnectionManager.cpp | 663 +++++++++++++++++- .../Source/GameNetwork/NetCommandMsg.cpp | 95 +++ .../GameNetwork/NetCommandWrapperList.cpp | 14 + .../Source/GameNetwork/NetPacketStructs.cpp | 130 ++++ .../GameEngine/Source/GameNetwork/Network.cpp | 105 +++ .../Source/GameNetwork/NetworkUtil.cpp | 4 + .../Source/Common/System/MemoryInit.cpp | 2 + GeneralsMD/Code/GameEngine/CMakeLists.txt | 2 + .../GameEngine/Include/GameLogic/GameLogic.h | 3 + .../GameNetwork/GeneralsOnline/MatchKeys.h | 77 ++ .../GameNetwork/GeneralsOnline/NGMPGame.h | 8 +- .../GameNetwork/GeneralsOnline/NGMP_include.h | 3 +- .../GameNetwork/GeneralsOnline/NetBytes.h | 18 + .../GameNetwork/GeneralsOnline/NetworkMesh.h | 31 + .../GeneralsOnline/OnlineServices_Init.h | 21 +- .../GameEngine/Source/Common/GameEngine.cpp | 8 + .../Source/Common/System/MemoryInit.cpp | 2 + .../GUICallbacks/Menus/WOLGameSetupMenu.cpp | 43 +- .../GameNetwork/GeneralsOnline/MatchKeys.cpp | 333 +++++++++ .../GameNetwork/GeneralsOnline/NGMPGame.cpp | 15 +- .../GeneralsOnline/NetworkMesh.cpp | 342 ++++++++- .../GeneralsOnline/NextGenTransport.cpp | 10 + .../OnlineServices_LobbyInterface.cpp | 4 + .../OnlineServices_RoomsInterface.cpp | 3 +- 31 files changed, 2166 insertions(+), 28 deletions(-) create mode 100644 GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/MatchKeys.h create mode 100644 GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/MatchKeys.cpp diff --git a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h index c5bee5502f2..7ec23934856 100644 --- a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h +++ b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h @@ -224,6 +224,9 @@ class ConnectionManager NetCommandList *m_pendingCommands; NetCommandList *m_relayedCommands; +#if !RETAIL_COMPATIBLE_NETWORKING + Int m_currentSenderSlot; ///< The slot of the peer the packet being read came from, -1 when not known. +#endif FrameMetrics m_frameMetrics; @@ -250,8 +253,82 @@ class ConnectionManager RecentCommandIds m_recentIds[MAX_SLOTS]; Bool isDuplicateCommand(NetCommandMsg *msg); + + Int slotForUserID(Int64 userID) const; + UnsignedInt m_formerRouterMask; ///< Slots that were the packet router in this match, their last packets can still be on the way. + Bool isCommandFromSender(NetCommandMsg *msg, Int senderSlot); ///< A command must come from the player it names, unless the packet router relays it + Bool noteSecurityDrop(); + Bool m_loggedSenderUnavailable; + Int m_securityDrops; void resetRecentCommandIds(); + // Frames with commands and chat that a peer passes on for another player need that player's signature. + // Every peer keeps the signatures it gets as long as the frames, so whoever holds a frame holds its proof. + enum { SIGNATURE_BYTES = 64, SIGNATURE_WAIT_MS = 3000, HELD_CHAT_COUNT = 8 }; + struct FrameSignature + { + Bool used; + UnsignedInt frame; + Bool relayed; ///< a command or the frame info arrived from a peer other than the author + Bool haveSignature; + Bool verified; + Int relayedBy; + time_t waitingSince; ///< when the frame was complete without a signature, 0 when not waiting + UnsignedByte signature[SIGNATURE_BYTES]; + }; + struct HeldChat + { + NetChatCommandMsg *msg; + Int relayedBy; + time_t since; + }; + struct ChatSignature + { + Bool used; + UnsignedInt author; + UnsignedShort commandID; + time_t since; + UnsignedByte signature[SIGNATURE_BYTES]; + }; + enum { RELAYED_WRAPPER_COUNT = 8 }; + struct RelayedWrapper + { + Bool used; + UnsignedShort wrappedCommandID; + Int relayedBy; + }; + std::vector m_frameSignatures[MAX_SLOTS]; ///< Indexed like the frame data, by frame modulo FRAME_DATA_LENGTH. + HeldChat m_heldChats[HELD_CHAT_COUNT]; + ChatSignature m_chatSignatures[HELD_CHAT_COUNT]; + RelayedWrapper m_relayedWrappers[MAX_SLOTS][RELAYED_WRAPPER_COUNT]; + Int m_nextRelayedWrapper[MAX_SLOTS]; + Int64 m_signerUserID[MAX_SLOTS]; ///< The user id per slot, kept after the player leaves to verify its last frames. + UnsignedInt m_lastFrameInfoSent; ///< The last frame our frame info went out for, our data up to it is final. + void clearSignatureState(Bool releaseChats); + void requestPassedOnFrameAgain(Int slot, UnsignedInt frame, Int passedOnBy); + FrameSignature *findFrameSignature(Int slot, UnsignedInt frame); + FrameSignature &frameSignatureFor(Int slot, UnsignedInt frame); + Int64 signerUserID(Int slot); + void noteFramePassedOn(Int author, UnsignedInt frame, Int relayedBy, Bool changesFrame); + void noteWrapperPassedOn(NetCommandMsg *wrapper, Int relayedBy); + Int wrapperPassedOnBy(NetCommandMsg *msg); + Bool buildSignedFrame(Int slot, UnsignedInt frame, std::vector &out); + Bool buildSignedCommand(Int slot, NetCommandMsg *msg, std::vector &out); + void signLocalFrame(NetFrameCommandMsg *frameInfo); + void signLocalChat(NetChatCommandMsg *msg); + Bool buildSignedLeave(Int slot, std::vector &out); +public: + Bool signLeaveNotice(NetDisconnectPlayerCommandMsg *msg); ///< The leaving player signs its leave notice. + Bool verifyLeaveNotice(NetDisconnectPlayerCommandMsg *msg); ///< The notice carries the signature of the player it says left. +private: + void attachStoredSignature(Int author, NetFrameCommandMsg *frameInfo); + Bool isPassedOnFrameAuthentic(Int slot, UnsignedInt frame); + void storeFrameSignature(Int author, UnsignedInt frame, const UnsignedByte *signature, Bool fromAuthor); + void processSignatures(NetSignaturesCommandMsg *msg); + Bool holdPassedOnChat(NetChatCommandMsg *msg); + void releaseHeldChat(Int index, const UnsignedByte *signature); + void expireHeldChats(time_t now); + ArrivalCushionStats m_cushionStats[MAX_SLOTS]; StallStats m_stallStats; Bool m_stallActive; diff --git a/Core/GameEngine/Include/GameNetwork/NetCommandMsg.h b/Core/GameEngine/Include/GameNetwork/NetCommandMsg.h index 49cd3bc21c3..e8d297a9ac0 100644 --- a/Core/GameEngine/Include/GameNetwork/NetCommandMsg.h +++ b/Core/GameEngine/Include/GameNetwork/NetCommandMsg.h @@ -269,10 +269,17 @@ class NetFrameCommandMsg : public NetCommandMsgT +{ + MEMORY_POOL_GLUE_WITH_USERLOOKUP_CREATE(NetSignedFrameCommandMsg, "NetSignedFrameCommandMsg") +public: + NetSignedFrameCommandMsg(); + NetSignedFrameCommandMsg(const NetFrameCommandMsg *frameMsg); + + UnsignedShort getCommandCount() const { return m_commandCount; } + void setCommandCount(UnsignedShort commandCount) { m_commandCount = commandCount; } + const UnsignedByte *getSignature() const { return m_signature; } + void setSignature(const UnsignedByte *signature); + + virtual Select getSmallNetPacketSelect() const override; + +protected: + UnsignedShort m_commandCount; + UnsignedByte m_signature[NET_SIGNATURE_SIZE]; +}; + +//----------------------------------------------------------------------------- +// The signatures of chat that the router relays, frame signatures ride inline in the frame info. +class NetSignaturesCommandMsg : public NetCommandMsgT +{ + MEMORY_POOL_GLUE_WITH_USERLOOKUP_CREATE(NetSignaturesCommandMsg, "NetSignaturesCommandMsg") +public: + enum { MAX_ENTRIES = 12 }; + typedef NetPacketSignaturesCommandData::Entry Entry; + + NetSignaturesCommandMsg(); + + Int getEntryCount() const { return m_entryCount; } + const Entry &getEntry(Int index) const { return m_entries[index]; } + Bool addEntry(UnsignedInt frame, UnsignedShort commandID, const UnsignedByte *signature); + + virtual Select getSmallNetPacketSelect() const override; + +protected: + Int m_entryCount; + Entry m_entries[MAX_ENTRIES]; +}; diff --git a/Core/GameEngine/Include/GameNetwork/NetPacketStructs.h b/Core/GameEngine/Include/GameNetwork/NetPacketStructs.h index c96a1e11b13..d2ebb2094a2 100644 --- a/Core/GameEngine/Include/GameNetwork/NetPacketStructs.h +++ b/Core/GameEngine/Include/GameNetwork/NetPacketStructs.h @@ -55,6 +55,8 @@ class NetDisconnectScreenOffCommandMsg; class NetFrameResendRequestCommandMsg; class NetLoadCompleteCommandMsg; class NetTimeOutGameStartCommandMsg; +class NetSignedFrameCommandMsg; +class NetSignaturesCommandMsg; //////////////////////////////////////////////////////////////////////////////// // Helper class to pass buffer pointer and size @@ -644,6 +646,8 @@ struct NetPacketDisconnectKeepAliveCommandBase // NetPacketDisconnectPlayerCommand //////////////////////////////////////////////////////////////////////////////// +enum { NET_SIGNATURE_SIZE = 64 }; + struct NetPacketDisconnectPlayerCommandData { typedef NetDisconnectPlayerCommandMsg CommandMsg; @@ -652,6 +656,10 @@ struct NetPacketDisconnectPlayerCommandData { UnsignedByte disconnectSlot; UnsignedInt disconnectFrame; +#if !RETAIL_COMPATIBLE_NETWORKING + UnsignedByte hasSignature; // the leaving player's signature of its leave, so peers can pass the notice on + UnsignedByte signature[NET_SIGNATURE_SIZE]; +#endif }; static size_t getSize(const NetCommandMsg &msg) { return sizeof(FixedData); } @@ -1177,6 +1185,82 @@ struct NetPacketFrameResendRequestCommandBase static size_t copyBytes(UnsignedByte *buffer, const NetCommandRef &ref); }; +//////////////////////////////////////////////////////////////////////////////// +// NetPacketSignedFrameCommand +//////////////////////////////////////////////////////////////////////////////// + +// A frame info followed by its author's signature. It is read back into a NetFrameCommandMsg. +struct NetPacketSignedFrameCommandData +{ + typedef NetSignedFrameCommandMsg CommandMsg; + + struct FixedData + { + UnsignedShort commandCount; + UnsignedByte signature[NET_SIGNATURE_SIZE]; + }; + + static size_t getSize(const NetCommandMsg &msg) { return sizeof(FixedData); } + static size_t copyBytes(UnsignedByte *buffer, const NetCommandRef &ref); + static size_t readMessage(NetCommandRef &ref, NetPacketBuf buf); + static size_t readIntoFrame(NetFrameCommandMsg *frameMsg, NetPacketBuf buf); +}; + +struct NetPacketSignedFrameCommandBase +{ + typedef NetSignedFrameCommandMsg CommandMsg; + + struct CommandBase + { + NetPacketCommandTypeField commandType; + NetPacketFrameField frame; + NetPacketRelayField relay; + NetPacketPlayerIdField playerId; + NetPacketCommandIdField commandId; + NetPacketDataField dataHeader; + }; + + static size_t getSize() { return sizeof(CommandBase); } + static size_t copyBytes(UnsignedByte *buffer, const NetCommandRef &ref); +}; + +//////////////////////////////////////////////////////////////////////////////// +// NetPacketSignaturesCommand +//////////////////////////////////////////////////////////////////////////////// + +struct NetPacketSignaturesCommandData +{ + typedef NetSignaturesCommandMsg CommandMsg; + + struct Entry + { + UnsignedInt frame; + UnsignedShort commandId; + UnsignedByte signature[NET_SIGNATURE_SIZE]; + }; + + static size_t getSize(const NetCommandMsg &msg); + static size_t copyBytes(UnsignedByte *buffer, const NetCommandRef &ref); + static size_t readMessage(NetCommandRef &ref, NetPacketBuf buf); +}; + +struct NetPacketSignaturesCommandBase +{ + typedef NetSignaturesCommandMsg CommandMsg; + + struct CommandBase + { + NetPacketCommandTypeField commandType; + NetPacketRelayField relay; + NetPacketPlayerIdField playerId; + NetPacketCommandIdField commandId; + NetPacketDataField dataHeader; + }; + + static size_t getSize() { return sizeof(CommandBase); } + static size_t copyBytes(UnsignedByte *buffer, const NetCommandRef &ref); +}; + //////////////////////////////////////////////////////////////////////////////// struct NetPacketAckCommand : public NetPacketCommandTemplate {}; @@ -1204,6 +1288,8 @@ struct NetPacketTimeOutGameStartCommand : public NetPacketCommandTemplat struct NetPacketDisconnectFrameCommand : public NetPacketCommandTemplate {}; struct NetPacketDisconnectScreenOffCommand : public NetPacketCommandTemplate {}; struct NetPacketFrameResendRequestCommand : public NetPacketCommandTemplate {}; +struct NetPacketSignedFrameCommand : public NetPacketCommandTemplate {}; +struct NetPacketSignaturesCommand : public NetPacketCommandTemplate {}; struct SmallNetPacketAckCommand : public SmallNetPacketCommandTemplate {}; struct SmallNetPacketFrameCommand : public SmallNetPacketCommandTemplate {}; @@ -1230,6 +1316,8 @@ struct SmallNetPacketTimeOutGameStartCommand : public SmallNetPacketCommandTe struct SmallNetPacketDisconnectFrameCommand : public SmallNetPacketCommandTemplate {}; struct SmallNetPacketDisconnectScreenOffCommand : public SmallNetPacketCommandTemplate {}; struct SmallNetPacketFrameResendRequestCommand : public SmallNetPacketCommandTemplate {}; +struct SmallNetPacketSignedFrameCommand : public SmallNetPacketCommandTemplate {}; +struct SmallNetPacketSignaturesCommand : public SmallNetPacketCommandTemplate {}; // Restore normal struct packing #pragma pack(pop) diff --git a/Core/GameEngine/Include/GameNetwork/NetworkDefs.h b/Core/GameEngine/Include/GameNetwork/NetworkDefs.h index af159f71874..4510ac4f336 100644 --- a/Core/GameEngine/Include/GameNetwork/NetworkDefs.h +++ b/Core/GameEngine/Include/GameNetwork/NetworkDefs.h @@ -133,6 +133,10 @@ struct TransportMessage Int length; UnsignedInt addr; UnsignedShort port; +#if defined(GENERALS_ONLINE) + Bool senderKnown; ///< The transport knows which user this packet came from, set by the receiving side only. + Int64 senderUserID; +#endif }; #pragma pack(pop) @@ -192,6 +196,9 @@ enum NetCommandType CPP_11(: Int) { NETCOMMANDTYPE_DISCONNECTFRAME, NETCOMMANDTYPE_DISCONNECTSCREENOFF, NETCOMMANDTYPE_DISCONNECTEND, + + NETCOMMANDTYPE_SIGNEDFRAMEINFO = 32, // A frame info with its author's signature, only on the wire. + NETCOMMANDTYPE_SIGNATURES, // Signatures of an author sent apart from its frames: for its chat. }; enum NetLocalStatus CPP_11(: Int) { diff --git a/Core/GameEngine/Include/GameNetwork/NetworkInterface.h b/Core/GameEngine/Include/GameNetwork/NetworkInterface.h index 2e28de9e1d2..a72a3a2c460 100644 --- a/Core/GameEngine/Include/GameNetwork/NetworkInterface.h +++ b/Core/GameEngine/Include/GameNetwork/NetworkInterface.h @@ -60,6 +60,9 @@ class NetworkInterface : public SubsystemInterface virtual void setLocalAddress(UnsignedInt ip, UnsignedInt port) = 0; ///< Tell the network what local ip and port to bind to. virtual Bool isFrameDataReady() = 0; ///< Are the commands for the next frame available? +#if defined(GENERALS_ONLINE) + virtual Bool isGameExitPending() const = 0; +#endif virtual Bool isStalling() = 0; virtual void parseUserList(const GameInfo* game) = 0; ///< Parse a userlist, creating connections virtual void startGame() = 0; ///< Sets the network game frame counter to -1 @@ -105,6 +108,8 @@ class NetworkInterface : public SubsystemInterface #if defined(GENERALS_ONLINE) virtual Bool getOverlayInfo(Int slot, NetworkOverlayInfo &info) = 0; virtual void SeedLatencyData(int highestLatency) = 0; + virtual void requireMatchKeys() {} + virtual Bool matchKeysRequired() const { return FALSE; } virtual bool IsSlugging() = 0; #endif diff --git a/Core/GameEngine/Source/GameLogic/System/GameLogicDispatch.cpp b/Core/GameEngine/Source/GameLogic/System/GameLogicDispatch.cpp index c9c7cbb7602..737e916da43 100644 --- a/Core/GameEngine/Source/GameLogic/System/GameLogicDispatch.cpp +++ b/Core/GameEngine/Source/GameLogic/System/GameLogicDispatch.cpp @@ -896,6 +896,21 @@ bool GameLogic::onNewGame(MAYBE_UNUSED GameMessage *msg) return true; } +#if defined(GENERALS_ONLINE) +void GameLogic::processPendingGameExit() +{ + for (GameMessage* msg = TheCommandList->getFirstMessage(); msg != nullptr; msg = msg->next()) { + if (msg->getType() == GameMessage::MSG_CLEAR_GAME_DATA) { + TheCommandList->removeMessage(msg); + deleteInstance(msg); + AIGroupPtr selectedGroup = nullptr; + onClearGameData(nullptr, selectedGroup); + return; + } + } +} +#endif + bool GameLogic::onClearGameData(MAYBE_UNUSED GameMessage *msg, AIGroupPtr ¤tlySelectedGroup) { #if defined(RTS_DEBUG) diff --git a/Core/GameEngine/Source/GameNetwork/Connection.cpp b/Core/GameEngine/Source/GameNetwork/Connection.cpp index 6df132bad1e..ffab6ac480f 100644 --- a/Core/GameEngine/Source/GameNetwork/Connection.cpp +++ b/Core/GameEngine/Source/GameNetwork/Connection.cpp @@ -157,6 +157,16 @@ void Connection::sendNetCommandMsg(NetCommandMsg *msg, UnsignedByte relay) { if (m_isQuitting) return; +#if defined(GENERALS_ONLINE) + // a signed frame info goes out in its signed form + if (msg->getNetCommandType() == NETCOMMANDTYPE_FRAMEINFO && ((NetFrameCommandMsg *)msg)->hasSignature()) { + NetSignedFrameCommandMsg *signedMsg = newInstance(NetSignedFrameCommandMsg)((NetFrameCommandMsg *)msg); + sendNetCommandMsg(signedMsg, relay); + signedMsg->detach(); + return; + } +#endif + if (m_netCommandList != nullptr) { // check to see if this command will fit in a packet. If not, we need to split it up. // we are splitting up the command here so that the retry logic will not try to diff --git a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp index 9d459b074a4..5237e9a9c32 100644 --- a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp +++ b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp @@ -59,6 +59,8 @@ #include "../NetworkMesh.h" #include "../ngmp_interfaces.h" +#include + #if defined(GENERALS_ONLINE) Int MIN_RUNAHEAD = 4; #endif @@ -211,6 +213,10 @@ static Bool hasValidTransferFileContent(const AsciiString& filePath, const Unsig */ ConnectionManager::~ConnectionManager() { +#if defined(GENERALS_ONLINE) + clearSignatureState(TRUE); +#endif + deleteInstance(m_localUser); m_localUser = nullptr; @@ -269,6 +275,9 @@ ConnectionManager::ConnectionManager() m_netCommandWrapperList = nullptr; m_localUser = nullptr; m_localUser = newInstance(User); +#if defined(GENERALS_ONLINE) + clearSignatureState(FALSE); +#endif } /** @@ -339,11 +348,18 @@ void ConnectionManager::init() m_latencyAverages[i] = 0.0; // using zero since all floating point standards should be able to specify 0.0 accurately. } m_smallestPacketArrivalCushion = -1; +#if !RETAIL_COMPATIBLE_NETWORKING + m_currentSenderSlot = -1; +#endif #if defined(GENERALS_ONLINE) m_stallActive = FALSE; m_stallStartTime = 0; m_stallMissingMask = 0; m_lastKnownRunAhead = 0; + m_loggedSenderUnavailable = FALSE; + m_securityDrops = 0; + m_formerRouterMask = 0; + clearSignatureState(TRUE); m_frameGroupingMs = 0; #endif @@ -415,6 +431,7 @@ void ConnectionManager::reset() m_netCommandWrapperList->reset(); #if defined(GENERALS_ONLINE) resetRecentCommandIds(); + clearSignatureState(TRUE); #endif m_localSlot = -1; @@ -537,11 +554,544 @@ Bool ConnectionManager::isDuplicateCommand(NetCommandMsg *msg) { return FALSE; } +Int ConnectionManager::slotForUserID(Int64 userID) const { + return (TheNGMPGame != nullptr) ? TheNGMPGame->getSlotOfUser(userID) : -1; +} + +// Counts a dropped command; TRUE for the drops that are logged (the first 20, then every 100th). +Bool ConnectionManager::noteSecurityDrop() { + ++m_securityDrops; + return m_securityDrops <= 20 || (m_securityDrops % 100) == 0; +} + +// How a command may reach us with another player's name on it: 0 only straight from its author, 1 authenticated by the +// author's signature (frames, chat), also from a former router, 2 only from the current packet router. +static Int relayClass(NetCommandType type) { + switch (type) { + case NETCOMMANDTYPE_FRAMEINFO: + case NETCOMMANDTYPE_SIGNEDFRAMEINFO: + case NETCOMMANDTYPE_GAMECOMMAND: + case NETCOMMANDTYPE_PLAYERLEAVE: + case NETCOMMANDTYPE_RUNAHEAD: + case NETCOMMANDTYPE_DESTROYPLAYER: + case NETCOMMANDTYPE_CHAT: + case NETCOMMANDTYPE_SIGNATURES: + return 1; + case NETCOMMANDTYPE_WRAPPER: + case NETCOMMANDTYPE_RUNAHEADMETRICS: + case NETCOMMANDTYPE_PROGRESS: + case NETCOMMANDTYPE_ACKBOTH: + case NETCOMMANDTYPE_ACKSTAGE1: + case NETCOMMANDTYPE_ACKSTAGE2: + return 2; + default: + return 0; // votes, disconnect frames, keep alives, load complete, file transfer, ... are never passed on + } +} + +// Checks that a command names its sender, or that the packet router or a former one passes it on. +Bool ConnectionManager::isCommandFromSender(NetCommandMsg *msg, Int senderSlot) { + const Int claimedSlot = (Int)msg->getPlayerID(); + if (claimedSlot == senderSlot && claimedSlot >= 0 && claimedSlot < MAX_SLOTS) { + return TRUE; + } + const Int cls = relayClass(msg->getNetCommandType()); + const Bool nameOk = claimedSlot >= 0 && claimedSlot < MAX_SLOTS && claimedSlot != (Int)m_localSlot; + Bool maySendForOthers = FALSE; + if (nameOk && cls == 1) { + maySendForOthers = (senderSlot == m_packetRouterSlot) || (senderSlot >= 0 && (m_formerRouterMask & (1u << senderSlot)) != 0); + } else if (nameOk && cls == 2) { + maySendForOthers = (senderSlot == m_packetRouterSlot); + } + // forwarded frames are checked in isPassedOnFrameAuthentic + if (maySendForOthers) { + return TRUE; + } + if (noteSecurityDrop()) { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] dropped %s claiming slot %d from slot %d (%d dropped so far)", + GetNetCommandTypeAsString(msg->getNetCommandType()), claimedSlot, (Int)senderSlot, m_securityDrops); + } + return FALSE; +} + void ConnectionManager::resetRecentCommandIds() { for (Int i = 0; i < MAX_SLOTS; ++i) { m_recentIds[i].valid = FALSE; } } + +static Int64 userIDOfSlot(Int slot) { + return (TheNGMPGame != nullptr) ? TheNGMPGame->getUserOfSlot(slot) : -1; +} + +static MatchKeys *activeMatchKeys() { + NetworkMesh *mesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + if (mesh == nullptr || !mesh->GetMatchKeys().IsActive()) { + return nullptr; + } + return &mesh->GetMatchKeys(); +} + +static void appendBytes(std::vector &out, UnsignedInt64 value, Int count) { + for (Int i = 0; i < count; ++i) { + out.push_back((UnsignedByte)(value >> (8 * i))); + } +} + +static void appendSignedHeader(std::vector &out, Char kind, UnsignedInt64 matchID, Int64 userID, Int slot) { + out.push_back('G'); + out.push_back('O'); + out.push_back('S'); + out.push_back((UnsignedByte)kind); + out.push_back(1); // version + appendBytes(out, matchID, 8); + appendBytes(out, (UnsignedInt64)userID, 8); + out.push_back((UnsignedByte)slot); +} + +// The command as the full packet format writes it, without the relay mask that differs per hop. +static void appendCommandBytes(std::vector &out, NetCommandMsg *msg) { + NetCommandRef *ref = NEW_NETCOMMANDREF(msg); + ref->setRelay(0); + const size_t start = out.size(); + const size_t size = msg->getSizeForNetPacket(); + appendBytes(out, 0, 4); + out.resize(start + 4 + size); + const size_t written = msg->copyBytesForNetPacket(&out[start + 4], *ref); + out.resize(start + 4 + written); + for (Int i = 0; i < 4; ++i) { + out[start + i] = (UnsignedByte)(written >> (8 * i)); + } + deleteInstance(ref); +} + +static Bool commandSignOrder(const NetCommandMsg *a, const NetCommandMsg *b) { + if (a->getNetCommandType() != b->getNetCommandType()) { + return a->getNetCommandType() < b->getNetCommandType(); + } + return a->getID() < b->getID(); +} + +void ConnectionManager::clearSignatureState(Bool releaseChats) { + for (Int slot = 0; slot < MAX_SLOTS; ++slot) { + m_frameSignatures[slot].assign(FRAME_DATA_LENGTH, FrameSignature()); + m_nextRelayedWrapper[slot] = 0; + m_signerUserID[slot] = -1; + for (Int wrapper = 0; wrapper < RELAYED_WRAPPER_COUNT; ++wrapper) { + m_relayedWrappers[slot][wrapper].used = FALSE; + } + } + for (Int held = 0; held < HELD_CHAT_COUNT; ++held) { + if (releaseChats && m_heldChats[held].msg != nullptr) { + m_heldChats[held].msg->detach(); + } + m_heldChats[held].msg = nullptr; + m_chatSignatures[held].used = FALSE; + } + m_currentSenderSlot = -1; + m_lastFrameInfoSent = 0; +} + +ConnectionManager::FrameSignature *ConnectionManager::findFrameSignature(Int slot, UnsignedInt frame) { + if (slot < 0 || slot >= MAX_SLOTS) { + return nullptr; + } + FrameSignature &entry = m_frameSignatures[slot][frame % FRAME_DATA_LENGTH]; + return (entry.used && entry.frame == frame) ? &entry : nullptr; +} + +ConnectionManager::FrameSignature &ConnectionManager::frameSignatureFor(Int slot, UnsignedInt frame) { + FrameSignature &entry = m_frameSignatures[slot][frame % FRAME_DATA_LENGTH]; + if (!entry.used || entry.frame != frame) { + entry = FrameSignature(); + entry.used = TRUE; + entry.frame = frame; + entry.relayedBy = -1; + } + return entry; +} + +// The user id of a slot, kept once seen so the last frames of a player that left can still be verified. +Int64 ConnectionManager::signerUserID(Int slot) { + if (slot < 0 || slot >= MAX_SLOTS) { + return -1; + } + const Int64 userID = userIDOfSlot(slot); + if (userID >= 0) { + m_signerUserID[slot] = userID; + } + return m_signerUserID[slot]; +} + +void ConnectionManager::noteFramePassedOn(Int author, UnsignedInt frame, Int relayedBy, Bool changesFrame) { + if (author < 0 || author >= MAX_SLOTS || author == (Int)m_localSlot) { + return; + } + if (TheGameLogic != nullptr && frame < TheGameLogic->getFrame()) { + return; + } + FrameSignature &entry = frameSignatureFor(author, frame); + entry.relayed = TRUE; + entry.relayedBy = relayedBy; + if (changesFrame) { + entry.verified = FALSE; + } +} + +void ConnectionManager::noteWrapperPassedOn(NetCommandMsg *wrapper, Int relayedBy) { + const Int author = (Int)wrapper->getPlayerID(); + if (author < 0 || author >= MAX_SLOTS) { + return; + } + const UnsignedShort wrappedID = ((NetWrapperCommandMsg *)wrapper)->getWrappedCommandID(); + for (Int i = 0; i < RELAYED_WRAPPER_COUNT; ++i) { + if (m_relayedWrappers[author][i].used && m_relayedWrappers[author][i].wrappedCommandID == wrappedID) { + m_relayedWrappers[author][i].relayedBy = relayedBy; + return; + } + } + RelayedWrapper &entry = m_relayedWrappers[author][m_nextRelayedWrapper[author]]; + m_nextRelayedWrapper[author] = (m_nextRelayedWrapper[author] + 1) % RELAYED_WRAPPER_COUNT; + entry.used = TRUE; + entry.wrappedCommandID = wrappedID; + entry.relayedBy = relayedBy; +} + +Int ConnectionManager::wrapperPassedOnBy(NetCommandMsg *msg) { + const Int author = (Int)msg->getPlayerID(); + if (author < 0 || author >= MAX_SLOTS) { + return -2; + } + for (Int i = 0; i < RELAYED_WRAPPER_COUNT; ++i) { + if (m_relayedWrappers[author][i].used && m_relayedWrappers[author][i].wrappedCommandID == msg->getID()) { + m_relayedWrappers[author][i].used = FALSE; + return m_relayedWrappers[author][i].relayedBy; + } + } + return author; +} + +/// Signed bytes: header (match, user id, slot), frame, command count, then the frame's commands in full packet form (relay 0) sorted by type and id. +/// The frame info id is not signed, a peer that passes it on renumbers it. +Bool ConnectionManager::buildSignedFrame(Int slot, UnsignedInt frame, std::vector &out) { + MatchKeys *keys = activeMatchKeys(); + const Int64 userID = signerUserID(slot); + if (keys == nullptr || userID < 0 || m_frameData[slot] == nullptr) { + return FALSE; + } + + std::vector commands; + NetCommandList *list = m_frameData[slot]->getFrameCommandList(frame); + for (NetCommandRef *ref = (list != nullptr) ? list->getFirstMessage() : nullptr; ref != nullptr; ref = ref->getNext()) { + commands.push_back(ref->getCommand()); + } + std::sort(commands.begin(), commands.end(), commandSignOrder); + + out.clear(); + appendSignedHeader(out, 'F', keys->GetMatchID(), userID, slot); + appendBytes(out, frame, 4); + appendBytes(out, m_frameData[slot]->getFrameCommandCount(frame), 2); + for (size_t i = 0; i < commands.size(); ++i) { + appendCommandBytes(out, commands[i]); + } + return TRUE; +} + +// What the author signs for its chat, the one command it sends through the packet router outside a frame. +Bool ConnectionManager::buildSignedCommand(Int slot, NetCommandMsg *msg, std::vector &out) { + MatchKeys *keys = activeMatchKeys(); + const Int64 userID = signerUserID(slot); + if (keys == nullptr || userID < 0) { + return FALSE; + } + out.clear(); + appendSignedHeader(out, 'C', keys->GetMatchID(), userID, slot); + appendCommandBytes(out, msg); + return TRUE; +} + +/// Signs a local frame that has commands; the signature rides in the frame info. Empty frames are not signed. +void ConnectionManager::signLocalFrame(NetFrameCommandMsg *frameInfo) { + MatchKeys *keys = activeMatchKeys(); + if (keys == nullptr || frameInfo->getCommandCount() == 0) { + return; + } + + std::vector data; + UnsignedByte signature[SIGNATURE_BYTES]; + if (!buildSignedFrame(m_localSlot, frameInfo->getExecutionFrame(), data) || !keys->Sign(&data[0], data.size(), signature)) { + return; + } + frameInfo->setSignature(signature); +} + +void ConnectionManager::signLocalChat(NetChatCommandMsg *chat) { + MatchKeys *keys = activeMatchKeys(); + if (keys == nullptr || m_packetRouterSlot >= MAX_SLOTS || m_packetRouterSlot == m_localSlot || m_connections[m_packetRouterSlot] == nullptr) { + return; + } + + std::vector data; + UnsignedByte signature[SIGNATURE_BYTES]; + if (!buildSignedCommand(m_localSlot, chat, data) || !keys->Sign(&data[0], data.size(), signature)) { + return; + } + + NetSignaturesCommandMsg *msg = newInstance(NetSignaturesCommandMsg); + msg->addEntry(chat->getExecutionFrame(), chat->getID(), signature); + msg->setPlayerID(m_localSlot); + msg->setID(GenerateNextCommandID()); + sendLocalCommand(msg, (UnsignedByte)(0xff & ~(1 << m_localSlot))); + msg->detach(); +} + +// What the leaver signs: the match, its user id and slot. A leave carries no frame, so a copy a peer passes on keeps its proof. +Bool ConnectionManager::buildSignedLeave(Int slot, std::vector &out) { + MatchKeys *keys = activeMatchKeys(); + const Int64 userID = signerUserID(slot); + if (keys == nullptr || userID < 0) { + return FALSE; + } + out.clear(); + appendSignedHeader(out, 'L', keys->GetMatchID(), userID, slot); + return TRUE; +} + +// The leaving player signs its own leave; without a key yet the notice goes out unsigned. +Bool ConnectionManager::signLeaveNotice(NetDisconnectPlayerCommandMsg *msg) { + MatchKeys *keys = activeMatchKeys(); + if (keys == nullptr || (Int)msg->getDisconnectSlot() != (Int)m_localSlot) { + return FALSE; + } + std::vector data; + UnsignedByte signature[SIGNATURE_BYTES]; + if (!buildSignedLeave(m_localSlot, data) || !keys->Sign(&data[0], data.size(), signature)) { + return FALSE; + } + msg->setSignature(signature); + return TRUE; +} + +// TRUE if the notice carries the signature of the player it says left. +Bool ConnectionManager::verifyLeaveNotice(NetDisconnectPlayerCommandMsg *msg) { + const Int slot = (Int)msg->getDisconnectSlot(); + MatchKeys *keys = activeMatchKeys(); + if (!msg->hasSignature() || slot < 0 || slot >= MAX_SLOTS || keys == nullptr) { + return FALSE; + } + std::vector data; + return buildSignedLeave(slot, data) && keys->Verify(signerUserID(slot), &data[0], data.size(), msg->getSignature()); +} + +void ConnectionManager::attachStoredSignature(Int author, NetFrameCommandMsg *frameInfo) { + FrameSignature *entry = findFrameSignature(author, frameInfo->getExecutionFrame()); + if (entry != nullptr && entry->haveSignature) { + frameInfo->setSignature(entry->signature); + } +} + +// the author's signature replaces a passed on one, never the reverse +void ConnectionManager::storeFrameSignature(Int author, UnsignedInt frame, const UnsignedByte *signature, Bool fromAuthor) { + if (author < 0 || author >= MAX_SLOTS || author == (Int)m_localSlot) { + return; + } + if (TheGameLogic != nullptr && frame + FRAMES_TO_KEEP < TheGameLogic->getFrame()) { + return; + } + FrameSignature &entry = frameSignatureFor(author, frame); + if (entry.haveSignature && !fromAuthor) { + return; + } + if (!entry.haveSignature || memcmp(entry.signature, signature, SIGNATURE_BYTES) != 0) { + memcpy(entry.signature, signature, SIGNATURE_BYTES); + entry.haveSignature = TRUE; + entry.verified = FALSE; + } +} + +// Asks the author for the frame again when we reach it, and the peer that passed it on, which holds it in any case. +void ConnectionManager::requestPassedOnFrameAgain(Int slot, UnsignedInt frame, Int passedOnBy) { + UnsignedByte targets = 0; + if (isPlayerConnected(slot)) { + targets |= (UnsignedByte)(1 << slot); + } + if (passedOnBy >= 0 && passedOnBy < MAX_SLOTS && passedOnBy != (Int)m_localSlot && isPlayerConnected(passedOnBy)) { + targets |= (UnsignedByte)(1 << passedOnBy); + } + if (targets == 0) { + requestFrameDataResend(slot, frame); + return; + } + + NetFrameResendRequestCommandMsg *msg = newInstance(NetFrameResendRequestCommandMsg); + msg->setPlayerID(m_localSlot); + msg->setFrameToResend(frame); + msg->setID(GenerateNextCommandID()); + sendLocalCommandDirect(msg, targets); + msg->detach(); +} + +/// A frame that a peer other than its author passed on runs only with the author's valid signature, else it is cleared and asked for again. +Bool ConnectionManager::isPassedOnFrameAuthentic(Int slot, UnsignedInt frame) { + FrameSignature *entry = findFrameSignature(slot, frame); + if (entry == nullptr || !entry->relayed || entry->verified || !TheNetwork->matchKeysRequired()) { + return TRUE; + } + // empty frames are not signed + if (m_frameData[slot]->getFrameCommandCount(frame) == 0) { + return TRUE; + } + + const char *problem = nullptr; + if (!entry->haveSignature) { + const time_t now = timeGetTime(); + if (entry->waitingSince == 0) { + entry->waitingSince = now; + } + if ((now - entry->waitingSince) < SIGNATURE_WAIT_MS) { + return FALSE; + } + problem = "unsigned"; + } else { + MatchKeys *keys = activeMatchKeys(); + std::vector data; + if (keys != nullptr && buildSignedFrame(slot, frame, data) && keys->Verify(signerUserID(slot), &data[0], data.size(), entry->signature)) { + entry->verified = TRUE; + return TRUE; + } + problem = "forged"; + } + + if (noteSecurityDrop()) { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] dropped %s commands for player %d relayed by %d (frame %u, %d dropped so far), asking for them again", + problem, slot, entry->relayedBy, frame, m_securityDrops); + } + const Int passedOnBy = entry->relayedBy; + m_frameData[slot]->resetFrame(frame, FALSE); + *entry = FrameSignature(); + entry->used = TRUE; + entry->frame = frame; + entry->relayedBy = -1; + requestPassedOnFrameAgain(slot, frame, passedOnBy); + return FALSE; +} + +void ConnectionManager::processSignatures(NetSignaturesCommandMsg *msg) { + const Int author = (Int)msg->getPlayerID(); + if (author < 0 || author >= MAX_SLOTS || author == (Int)m_localSlot) { + return; + } + + const time_t now = timeGetTime(); + for (Int e = 0; e < msg->getEntryCount(); ++e) { + const NetSignaturesCommandMsg::Entry &signedEntry = msg->getEntry(e); + expireHeldChats(now); + Bool released = FALSE; + for (Int held = 0; held < HELD_CHAT_COUNT && !released; ++held) { + NetChatCommandMsg *chat = m_heldChats[held].msg; + if (chat != nullptr && (Int)chat->getPlayerID() == author && chat->getID() == signedEntry.commandId) { + releaseHeldChat(held, signedEntry.signature); + released = TRUE; + } + } + if (released) { + continue; + } + + // the chat is not here yet, keep the signature for it, the oldest one gives way + Int index = 0; + for (Int i = 0; i < HELD_CHAT_COUNT; ++i) { + if (!m_chatSignatures[i].used) { + index = i; + break; + } + if (m_chatSignatures[i].since < m_chatSignatures[index].since) { + index = i; + } + } + ChatSignature &stored = m_chatSignatures[index]; + stored.used = TRUE; + stored.author = (UnsignedInt)author; + stored.commandID = signedEntry.commandId; + stored.since = now; + memcpy(stored.signature, signedEntry.signature, SIGNATURE_BYTES); + } +} + +// Chat the packet router relays is held until its author's signature checks out. +Bool ConnectionManager::holdPassedOnChat(NetChatCommandMsg *msg) { + const Int author = (Int)msg->getPlayerID(); + const Int relayedBy = m_currentSenderSlot; + if (relayedBy < 0 || author == relayedBy || author == (Int)m_localSlot || !TheNetwork->matchKeysRequired()) { + return FALSE; + } + + const time_t now = timeGetTime(); + expireHeldChats(now); + + Int index = 0; + for (Int i = 0; i < HELD_CHAT_COUNT; ++i) { + if (m_heldChats[i].msg == nullptr) { + index = i; + break; + } + if (m_heldChats[i].since < m_heldChats[index].since) { + index = i; + } + } + if (m_heldChats[index].msg != nullptr) { + ++m_securityDrops; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] dropped unsigned chat for player %d relayed by %d, too many chats wait for a signature", + (Int)m_heldChats[index].msg->getPlayerID(), m_heldChats[index].relayedBy); + m_heldChats[index].msg->detach(); + m_heldChats[index].msg = nullptr; + } + msg->attach(); + m_heldChats[index].msg = msg; + m_heldChats[index].relayedBy = relayedBy; + m_heldChats[index].since = now; + + for (Int i = 0; i < HELD_CHAT_COUNT; ++i) { + if (m_chatSignatures[i].used && m_chatSignatures[i].author == (UnsignedInt)author && m_chatSignatures[i].commandID == msg->getID()) { + m_chatSignatures[i].used = FALSE; + releaseHeldChat(index, m_chatSignatures[i].signature); + break; + } + } + return TRUE; +} + +void ConnectionManager::releaseHeldChat(Int index, const UnsignedByte *signature) { + NetChatCommandMsg *chat = m_heldChats[index].msg; + const Int relayedBy = m_heldChats[index].relayedBy; + m_heldChats[index].msg = nullptr; + + const Int author = (Int)chat->getPlayerID(); + MatchKeys *keys = activeMatchKeys(); + std::vector data; + if (keys != nullptr && buildSignedCommand(author, chat, data) && keys->Verify(signerUserID(author), &data[0], data.size(), signature)) { + processChat(chat); + } else { + ++m_securityDrops; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] dropped forged chat for player %d relayed by %d", author, relayedBy); + } + chat->detach(); +} + +void ConnectionManager::expireHeldChats(time_t now) { + for (Int i = 0; i < HELD_CHAT_COUNT; ++i) { + if (m_heldChats[i].msg != nullptr && (now - m_heldChats[i].since) > SIGNATURE_WAIT_MS * 3) { + ++m_securityDrops; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] dropped unsigned chat for player %d relayed by %d", + (Int)m_heldChats[i].msg->getPlayerID(), m_heldChats[i].relayedBy); + m_heldChats[i].msg->detach(); + m_heldChats[i].msg = nullptr; + } + if (m_chatSignatures[i].used && (now - m_chatSignatures[i].since) > SIGNATURE_WAIT_MS * 3) { + m_chatSignatures[i].used = FALSE; + } + } +} #endif /** @@ -566,11 +1116,40 @@ void ConnectionManager::doRelay() { // Get the command list from the packet. NetCommandList *cmdList = packet->getCommandList(); +#if defined(GENERALS_ONLINE) + // -2 when the transport cannot say who sent the packet + Int senderSlot = -2; + if (m_transport->m_inBuffer[i].senderKnown) { + senderSlot = slotForUserID(m_transport->m_inBuffer[i].senderUserID); + if (senderSlot < 0) { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] dropped a packet of %d commands from user %lld, who is not in this game", (Int)packet->getNumCommands(), (long long)m_transport->m_inBuffer[i].senderUserID); + } + } else if (!m_loggedSenderUnavailable) { + m_loggedSenderUnavailable = TRUE; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] the transport does not say who sent a packet (anticheat plugin path), commands are not checked against their sender"); + } +#endif + +#if !RETAIL_COMPATIBLE_NETWORKING +#if defined(GENERALS_ONLINE) + m_currentSenderSlot = (senderSlot >= 0) ? senderSlot : -1; +#endif +#endif + // Iterate through the commands in this packet and send them to the proper connections. for (NetCommandRef* cmd = cmdList->getFirstMessage(); cmd; cmd = cmd->getNext()) { //DEBUG_LOG(("ConnectionManager::doRelay() - Looking at a command of type %s", //GetNetCommandTypeAsString(cmd->getCommand()->getNetCommandType()))); +#if defined(GENERALS_ONLINE) + if (senderSlot == -1 || (senderSlot >= 0 && !isCommandFromSender(cmd->getCommand(), senderSlot))) { + continue; // not acked and not processed + } + if (senderSlot >= 0 && cmd->getCommand()->getNetCommandType() == NETCOMMANDTYPE_WRAPPER && (Int)cmd->getCommand()->getPlayerID() != senderSlot) { + noteWrapperPassedOn(cmd->getCommand(), senderSlot); + } +#endif + if (CommandRequiresAck(cmd->getCommand())) { ackCommand(cmd, m_localSlot); #if defined(GENERALS_ONLINE) @@ -583,6 +1162,9 @@ void ConnectionManager::doRelay() { sendRemoteCommand(cmd); } } +#if defined(GENERALS_ONLINE) + m_currentSenderSlot = -1; +#endif // Delete this packet since we won't be needing it anymore. deleteInstance(packet); @@ -598,8 +1180,20 @@ void ConnectionManager::doRelay() { } } +#if !RETAIL_COMPATIBLE_NETWORKING + m_currentSenderSlot = -1; +#endif + NetCommandList *cmdList = m_netCommandWrapperList->getReadyCommands(); for (NetCommandRef* cmd = cmdList->getFirstMessage(); cmd; cmd = cmd->getNext()) { +#if defined(GENERALS_ONLINE) + // a command put together from pieces some other peer passed on counts as passed on by that peer + m_currentSenderSlot = wrapperPassedOnBy(cmd->getCommand()); + if (m_currentSenderSlot < 0 || !isCommandFromSender(cmd->getCommand(), m_currentSenderSlot)) { + m_currentSenderSlot = -1; + continue; // the command inside the pieces is held to the same rule as one sent plain + } +#endif if (CommandRequiresAck(cmd->getCommand())) { ackCommand(cmd, m_localSlot); #if defined(GENERALS_ONLINE) @@ -612,6 +1206,9 @@ void ConnectionManager::doRelay() { sendRemoteCommand(cmd); } } +#if defined(GENERALS_ONLINE) + m_currentSenderSlot = -1; +#endif // Delete this packet since we won't be needing it anymore. deleteInstance(packet); @@ -721,9 +1318,20 @@ Bool ConnectionManager::processNetCommand(NetCommandRef *ref) { return FALSE; case NETCOMMANDTYPE_CHAT: +#if defined(GENERALS_ONLINE) + if (holdPassedOnChat((NetChatCommandMsg *)msg)) { + return FALSE; // shown once the author's signature checks out + } +#endif processChat((NetChatCommandMsg *)msg); return FALSE; +#if defined(GENERALS_ONLINE) + case NETCOMMANDTYPE_SIGNATURES: + processSignatures((NetSignaturesCommandMsg *)msg); + return FALSE; +#endif + case NETCOMMANDTYPE_FILE: processFile((NetFileCommandMsg *)msg); return FALSE; @@ -1077,6 +1685,14 @@ void ConnectionManager::processFrameInfo(NetFrameCommandMsg *msg) { if (playerID < MAX_SLOTS) { if (m_frameData[playerID] != nullptr) { +#if defined(GENERALS_ONLINE) + if (msg->hasSignature()) { + storeFrameSignature((Int)playerID, msg->getExecutionFrame(), msg->getSignature(), m_currentSenderSlot == (Int)playerID); + } + if (m_currentSenderSlot >= 0 && (Int)playerID != m_currentSenderSlot) { + noteFramePassedOn((Int)playerID, msg->getExecutionFrame(), m_currentSenderSlot, m_frameData[playerID]->getFrameCommandCount(msg->getExecutionFrame()) != msg->getCommandCount()); + } +#endif // DEBUG_LOG(("ConnectionManager::processFrameInfo - player %d, frame %d, command count %d, received on frame %d", playerID, msg->getExecutionFrame(), msg->getCommandCount(), TheGameLogic->getFrame())); #if defined(GENERALS_ONLINE) // a redundant copy of a frame that was already seen says nothing about how early frames arrive @@ -1391,6 +2007,11 @@ void ConnectionManager::sendRemoteCommand(NetCommandRef *msg) { if (IsCommandSynchronized(msg->getCommand()->getNetCommandType())) { DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("ConnectionManager::sendRemoteCommand - adding net command of type %s to player %d for frame %d", GetNetCommandTypeAsString(msg->getCommand()->getNetCommandType()), msg->getCommand()->getPlayerID(), msg->getCommand()->getExecutionFrame())); isNew = frameDataMgr->addNetCommandMsg(msg->getCommand()); +#if defined(GENERALS_ONLINE) + if (isNew && m_currentSenderSlot >= 0 && (Int)playerID != m_currentSenderSlot) { + noteFramePassedOn((Int)playerID, msg->getCommand()->getExecutionFrame(), m_currentSenderSlot, TRUE); + } +#endif } } @@ -1883,7 +2504,14 @@ void ConnectionManager::processFrameTick(UnsignedInt frame) { DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("ConnectionManager::processFrameTick - sending frame info for frame %d, ID %d, command count %d", frame, msg->getID(), commandCount)); +#if defined(GENERALS_ONLINE) + signLocalFrame(msg); +#endif + sendLocalCommand(msg, 0xff & ~(1 << m_localSlot)); +#if defined(GENERALS_ONLINE) + m_lastFrameInfoSent = frame; +#endif msg->detach(); } @@ -2052,6 +2680,11 @@ Bool ConnectionManager::allCommandsReady(UnsignedInt frame, Bool justTesting /* requestFrameDataResend(i, frame); retval = FALSE; } +#if defined(GENERALS_ONLINE) + else if (i != (Int)m_localSlot && !isPassedOnFrameAuthentic(i, frame)) { + retval = FALSE; + } +#endif } } @@ -2060,6 +2693,15 @@ Bool ConnectionManager::allCommandsReady(UnsignedInt frame, Bool justTesting /* for (i = 0; i < MAX_SLOTS; ++i) { if ((m_frameData[i] != nullptr) && (i != m_localSlot)) { m_frameData[i]->resetFrame(frame, FALSE); +#if defined(GENERALS_ONLINE) + // the resent copy is judged by where it comes from, a signature that came already stays + FrameSignature *entry = findFrameSignature(i, frame); + if (entry != nullptr) { + entry->relayed = FALSE; + entry->verified = FALSE; + entry->waitingSince = 0; + } +#endif } } } @@ -2225,13 +2867,12 @@ PlayerLeaveCode ConnectionManager::disconnectPlayer(Int slot) { // } if (slot == m_packetRouterSlot) { - Int index = 0; - while ((index < MAX_SLOTS) && (m_packetRouterFallback[index] != m_packetRouterSlot)) { - ++index; - } - ++index; - if (index < MAX_SLOTS) { - m_packetRouterSlot = m_packetRouterFallback[index]; +#if defined(GENERALS_ONLINE) + m_formerRouterMask |= (1u << m_packetRouterSlot); +#endif + const UnsignedInt nextRouter = getNextPacketRouterSlot(m_packetRouterSlot); + if (nextRouter < (UnsignedInt)MAX_SLOTS) { + m_packetRouterSlot = nextRouter; } else { DEBUG_LOG(("ConnectionManager::disconnectPlayer - packet router had no valid fallback, defaulting to local slot %d", m_localSlot)); m_packetRouterSlot = m_localSlot; @@ -2620,6 +3261,9 @@ void ConnectionManager::sendChat(UnicodeString text, Int playerMask, UnsignedInt DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("Chat message has ID of %d, mask of %8.8X, text of %ls", msg->getID(), msg->getPlayerMask(), msg->getText().str())); sendLocalCommand(msg, 0xff ^ (1 << m_localSlot)); +#if defined(GENERALS_ONLINE) + signLocalChat(msg); +#endif processChat(msg); msg->detach(); @@ -2946,6 +3590,11 @@ void ConnectionManager::sendSingleFrameToPlayer(UnsignedInt playerID, UnsignedIn } msg->setPlayerID(i); DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("ConnectionManager::sendFrameDataToPlayer - sending frame info from player %d to player %d for frame %d with command count %d and ID %d and relay %d", i, playerID, msg->getExecutionFrame(), msg->getCommandCount(), msg->getID(), relay)); +#if defined(GENERALS_ONLINE) + if (i != (Int)m_localSlot) { + attachStoredSignature(i, msg); + } +#endif sendLocalCommandDirect(msg, relay); msg->detach(); } diff --git a/Core/GameEngine/Source/GameNetwork/NetCommandMsg.cpp b/Core/GameEngine/Source/GameNetwork/NetCommandMsg.cpp index a931a33a554..2dead32b164 100644 --- a/Core/GameEngine/Source/GameNetwork/NetCommandMsg.cpp +++ b/Core/GameEngine/Source/GameNetwork/NetCommandMsg.cpp @@ -27,6 +27,7 @@ #include "GameNetwork/NetCommandMsg.h" #include "GameNetwork/NetPacketStructs.h" +#include "GameNetwork/NetCommandRef.h" #include "GameNetwork/GameMessageParser.h" #include "Common/GameState.h" #include "Common/PlayerList.h" @@ -346,6 +347,24 @@ NetAckStage2CommandMsg::~NetAckStage2CommandMsg() { NetFrameCommandMsg::NetFrameCommandMsg() { m_commandCount = 0; m_commandType = NETCOMMANDTYPE_FRAMEINFO; + m_hasSignature = FALSE; + memset(m_signature, 0, sizeof(m_signature)); +} + +void NetFrameCommandMsg::setSignature(const UnsignedByte *signature) { + memcpy(m_signature, signature, sizeof(m_signature)); + m_hasSignature = TRUE; +} + +// A frame info that came in its signed form is read with its signature and is a plain frame info from then on. +size_t NetFrameCommandMsg::readMessageData(NetCommandRef& ref, NetPacketBuf buf) const { + if (m_commandType == NETCOMMANDTYPE_SIGNEDFRAMEINFO) { + NetFrameCommandMsg *frameMsg = static_cast(ref.getCommand()); + const size_t size = NetPacketSignedFrameCommandData::readIntoFrame(frameMsg, buf); + frameMsg->setNetCommandType(NETCOMMANDTYPE_FRAMEINFO); + return size; + } + return SmallNetPacketFrameCommand::CommandData::readMessage(ref, buf); } /** @@ -613,6 +632,9 @@ NetCommandMsg::Select NetDisconnectKeepAliveCommandMsg::getSmallNetPacketSelect( NetDisconnectPlayerCommandMsg::NetDisconnectPlayerCommandMsg() { m_commandType = NETCOMMANDTYPE_DISCONNECTPLAYER; m_disconnectSlot = 0; + m_disconnectFrame = 0; + m_hasSignature = FALSE; + memset(m_signature, 0, sizeof(m_signature)); } /** @@ -635,6 +657,11 @@ void NetDisconnectPlayerCommandMsg::setDisconnectSlot(UnsignedByte slot) { m_disconnectSlot = slot; } +void NetDisconnectPlayerCommandMsg::setSignature(const UnsignedByte *signature) { + memcpy(m_signature, signature, sizeof(m_signature)); + m_hasSignature = TRUE; +} + /** * Sets the disconnect frame */ @@ -1253,3 +1280,71 @@ NetCommandMsg::Select NetTimeOutGameStartCommandMsg::getSmallNetPacketSelect() c select.useCommandId = 1; return select; } + +//------------------------- +// NetSignedFrameCommandMsg +//------------------------- +NetSignedFrameCommandMsg::NetSignedFrameCommandMsg() { + m_commandType = NETCOMMANDTYPE_SIGNEDFRAMEINFO; + m_commandCount = 0; + memset(m_signature, 0, sizeof(m_signature)); +} + +NetSignedFrameCommandMsg::NetSignedFrameCommandMsg(const NetFrameCommandMsg *frameMsg) { + m_commandType = NETCOMMANDTYPE_SIGNEDFRAMEINFO; + m_executionFrame = frameMsg->getExecutionFrame(); + m_playerID = frameMsg->getPlayerID(); + m_id = frameMsg->getID(); + m_commandCount = frameMsg->getCommandCount(); + memcpy(m_signature, frameMsg->getSignature(), sizeof(m_signature)); +} + +NetSignedFrameCommandMsg::~NetSignedFrameCommandMsg() { +} + +void NetSignedFrameCommandMsg::setSignature(const UnsignedByte *signature) { + memcpy(m_signature, signature, sizeof(m_signature)); +} + +NetCommandMsg::Select NetSignedFrameCommandMsg::getSmallNetPacketSelect() const { + Select select; + select.useCommandType = 1; + select.useRelay = 1; + select.useFrame = 1; + select.usePlayerId = 1; + select.useCommandId = 1; + return select; +} + +//------------------------- +// NetSignaturesCommandMsg +//------------------------- +NetSignaturesCommandMsg::NetSignaturesCommandMsg() { + m_commandType = NETCOMMANDTYPE_SIGNATURES; + m_entryCount = 0; + memset(m_entries, 0, sizeof(m_entries)); +} + +NetSignaturesCommandMsg::~NetSignaturesCommandMsg() { +} + +Bool NetSignaturesCommandMsg::addEntry(UnsignedInt frame, UnsignedShort commandID, const UnsignedByte *signature) { + if (m_entryCount >= MAX_ENTRIES) { + return FALSE; + } + Entry &entry = m_entries[m_entryCount++]; + entry.frame = frame; + entry.commandId = commandID; + memcpy(entry.signature, signature, sizeof(entry.signature)); + return TRUE; +} + +NetCommandMsg::Select NetSignaturesCommandMsg::getSmallNetPacketSelect() const { + Select select; + select.useCommandType = 1; + select.useRelay = 1; + select.useFrame = 0; + select.usePlayerId = 1; + select.useCommandId = 1; + return select; +} diff --git a/Core/GameEngine/Source/GameNetwork/NetCommandWrapperList.cpp b/Core/GameEngine/Source/GameNetwork/NetCommandWrapperList.cpp index 79aa405544d..db9422ff5c4 100644 --- a/Core/GameEngine/Source/GameNetwork/NetCommandWrapperList.cpp +++ b/Core/GameEngine/Source/GameNetwork/NetCommandWrapperList.cpp @@ -29,6 +29,7 @@ #include "GameNetwork/NetCommandWrapperList.h" #include "GameNetwork/NetPacket.h" +#include "../NGMP_include.h" //////////////////////////////////////////////////////////////////////////////////////////////////// ////// NetCommandWrapperListNode /////////////////////////////////////////////////////////////////// @@ -233,6 +234,19 @@ NetCommandList * NetCommandWrapperList::getReadyCommands() next = temp->m_next; if (temp->isComplete()) { NetCommandRef *msg = NetPacket::ConstructNetCommandMsgFromRawData(temp->getRawData(), temp->getRawDataLength()); + +#if defined(GENERALS_ONLINE) + // the wrapper was checked against its sender, the command inside has to name the same player + if (msg != nullptr && msg->getCommand() != nullptr && msg->getCommand()->getPlayerID() != temp->getPlayerID()) { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] dropped a wrapped command claiming slot %d inside a wrapper from slot %d", (Int)msg->getCommand()->getPlayerID(), (Int)temp->getPlayerID()); + deleteInstance(msg); + noteCompleted(temp->getPlayerID(), temp->getCommandID()); + removeFromList(temp); + temp = next; + continue; + } +#endif + NetCommandRef *ret = retlist->addMessage(msg->getCommand()); ret->setRelay(msg->getRelay()); diff --git a/Core/GameEngine/Source/GameNetwork/NetPacketStructs.cpp b/Core/GameEngine/Source/GameNetwork/NetPacketStructs.cpp index 500fcdbee88..846d1443738 100644 --- a/Core/GameEngine/Source/GameNetwork/NetPacketStructs.cpp +++ b/Core/GameEngine/Source/GameNetwork/NetPacketStructs.cpp @@ -241,6 +241,14 @@ NetCommandMsg *SmallNetPacketCommandBase::constructNetCommandMsg(const CommandBa case NETCOMMANDTYPE_FRAMERESENDREQUEST: msg = newInstance(NetFrameResendRequestCommandMsg); break; + case NETCOMMANDTYPE_SIGNEDFRAMEINFO: + // read back into a plain frame info that keeps the signature, see NetFrameCommandMsg::readMessageData + msg = newInstance(NetFrameCommandMsg); + msg->setNetCommandType(NETCOMMANDTYPE_SIGNEDFRAMEINFO); + break; + case NETCOMMANDTYPE_SIGNATURES: + msg = newInstance(NetSignaturesCommandMsg); + break; default: DEBUG_CRASH(("SmallNetPacketCommandBase::constructNetCommandMsg: Unexpected command type '%d' encountered.", commandType)); return nullptr; @@ -538,6 +546,10 @@ size_t NetPacketDisconnectPlayerCommandData::copyBytes(UnsignedByte *buffer, con FixedData data; data.disconnectSlot = cmdMsg->getDisconnectSlot(); data.disconnectFrame = cmdMsg->getDisconnectFrame(); +#if !RETAIL_COMPATIBLE_NETWORKING + data.hasSignature = cmdMsg->hasSignature() ? 1 : 0; + memcpy(data.signature, cmdMsg->getSignature(), sizeof(data.signature)); +#endif return network::writeObject(buffer, data); } @@ -548,10 +560,19 @@ size_t NetPacketDisconnectPlayerCommandData::readMessage(NetCommandRef &ref, Net FixedData data; data.disconnectSlot = 0; data.disconnectFrame = 0; +#if !RETAIL_COMPATIBLE_NETWORKING + data.hasSignature = 0; + memset(data.signature, 0, sizeof(data.signature)); +#endif size_t size = network::readObject(data, buf); cmdMsg->setDisconnectSlot(data.disconnectSlot); cmdMsg->setDisconnectFrame(data.disconnectFrame); +#if !RETAIL_COMPATIBLE_NETWORKING + if (data.hasSignature != 0) { + cmdMsg->setSignature(data.signature); + } +#endif return size; } @@ -1428,3 +1449,112 @@ size_t NetPacketFrameResendRequestCommandBase::copyBytes(UnsignedByte *buffer, c return network::writeObject(buffer, base); } + +//////////////////////////////////////////////////////////////////////////////// +// NetPacketSignedFrameCommand +//////////////////////////////////////////////////////////////////////////////// + +size_t NetPacketSignedFrameCommandData::copyBytes(UnsignedByte *buffer, const NetCommandRef &ref) +{ + const CommandMsg *cmdMsg = static_cast(ref.getCommand()); + FixedData data; + data.commandCount = cmdMsg->getCommandCount(); + memcpy(data.signature, cmdMsg->getSignature(), NET_SIGNATURE_SIZE); + + return network::writeObject(buffer, data); +} + +size_t NetPacketSignedFrameCommandData::readMessage(NetCommandRef &ref, NetPacketBuf buf) +{ + CommandMsg *cmdMsg = static_cast(ref.getCommand()); + FixedData data; + memset(&data, 0, sizeof(data)); + + size_t size = network::readObject(data, buf); + cmdMsg->setCommandCount(data.commandCount); + cmdMsg->setSignature(data.signature); + + return size; +} + +size_t NetPacketSignedFrameCommandData::readIntoFrame(NetFrameCommandMsg *frameMsg, NetPacketBuf buf) +{ + FixedData data; + memset(&data, 0, sizeof(data)); + + size_t size = network::readObject(data, buf); + frameMsg->setCommandCount(data.commandCount); + frameMsg->setSignature(data.signature); + + return size; +} + +size_t NetPacketSignedFrameCommandBase::copyBytes(UnsignedByte *buffer, const NetCommandRef &ref) +{ + const NetCommandMsg *msg = ref.getCommand(); + CommandBase base; + base.commandType.commandType = msg->getNetCommandType(); + base.relay.relay = ref.getRelay(); + base.frame.frame = msg->getExecutionFrame(); + base.playerId.playerId = msg->getPlayerID(); + base.commandId.commandId = msg->getID(); + + return network::writeObject(buffer, base); +} + +//////////////////////////////////////////////////////////////////////////////// +// NetPacketSignaturesCommand +//////////////////////////////////////////////////////////////////////////////// + +size_t NetPacketSignaturesCommandData::getSize(const NetCommandMsg &msg) +{ + const CommandMsg *cmdMsg = static_cast(&msg); + return sizeof(UnsignedByte) + cmdMsg->getEntryCount() * sizeof(Entry); +} + +size_t NetPacketSignaturesCommandData::copyBytes(UnsignedByte *buffer, const NetCommandRef &ref) +{ + const CommandMsg *cmdMsg = static_cast(ref.getCommand()); + + size_t size = 0; + size += network::writePrimitive(buffer + size, (UnsignedByte)cmdMsg->getEntryCount()); + for (Int i = 0; i < cmdMsg->getEntryCount(); ++i) + { + size += network::writeObject(buffer + size, cmdMsg->getEntry(i)); + } + return size; +} + +size_t NetPacketSignaturesCommandData::readMessage(NetCommandRef &ref, NetPacketBuf buf) +{ + CommandMsg *cmdMsg = static_cast(ref.getCommand()); + UnsignedByte count = 0; + + size_t size = 0; + size += network::readObject(count, buf.offset(size)); + for (Int i = 0; i < (Int)count; ++i) + { + Entry entry; + memset(&entry, 0, sizeof(entry)); + const size_t read = network::readObject(entry, buf.offset(size)); + if (read < sizeof(entry)) + { + break; + } + size += read; + cmdMsg->addEntry(entry.frame, entry.commandId, entry.signature); + } + return size; +} + +size_t NetPacketSignaturesCommandBase::copyBytes(UnsignedByte *buffer, const NetCommandRef &ref) +{ + const NetCommandMsg *msg = ref.getCommand(); + CommandBase base; + base.commandType.commandType = msg->getNetCommandType(); + base.relay.relay = ref.getRelay(); + base.playerId.playerId = msg->getPlayerID(); + base.commandId.commandId = msg->getID(); + + return network::writeObject(buffer, base); +} diff --git a/Core/GameEngine/Source/GameNetwork/Network.cpp b/Core/GameEngine/Source/GameNetwork/Network.cpp index d5e6b6ada93..ea6d13f9f83 100644 --- a/Core/GameEngine/Source/GameNetwork/Network.cpp +++ b/Core/GameEngine/Source/GameNetwork/Network.cpp @@ -52,6 +52,7 @@ #include "Common/Recorder.h" #include "GameClient/MessageBox.h" #include "../NGMP_include.h" +#include "../ngmp_interfaces.h" #if defined(DEBUG_CRC) && !RETAIL_COMPATIBLE_NETWORKING @@ -114,6 +115,8 @@ class Network : public NetworkInterface #if defined(GENERALS_ONLINE) virtual void SeedLatencyData(int latency); + virtual void requireMatchKeys() override { m_matchKeysRequired = TRUE; } + virtual Bool matchKeysRequired() const override { return m_matchKeysRequired; } #endif void setLocalAddress(UnsignedInt ip, UnsignedInt port); @@ -121,6 +124,9 @@ class Network : public NetworkInterface UnsignedInt getFrameRate(void) { return m_frameRate; } UnsignedInt getPacketArrivalCushion(void); ///< Returns the smallest packet arrival cushion since this was last called. Bool isFrameDataReady(void); +#if defined(GENERALS_ONLINE) + virtual Bool isGameExitPending() const override { return m_localStatus == NETLOCALSTATUS_POSTGAME; } +#endif virtual Bool isStalling(); void parseUserList(const GameInfo* game); void startGame(void); ///< Sets the network game frame counter to -1 @@ -239,6 +245,13 @@ class Network : public NetworkInterface #if defined(RTS_DEBUG) Bool m_networkOn; #endif + +#if defined(GENERALS_ONLINE) + Bool m_matchKeysRequired; + Bool m_matchStartFailed; + Bool startPrerequisitesReady() const; + void failMatchStart(); ///< Leaves rather than starting without authentication. +#endif }; UnsignedInt Network::getPingFrame() @@ -311,6 +324,15 @@ Network::~Network() */ Bool Network::deinit() { +#if defined(GENERALS_ONLINE) + if (m_matchKeysRequired) { + NetworkMesh* mesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + if (mesh != nullptr) { + mesh->EndMatchKeys(); + } + m_matchKeysRequired = FALSE; + } +#endif if (m_conMgr) { m_conMgr->destroyGameMessages(); @@ -357,6 +379,11 @@ void Network::init() m_localStatus = NETLOCALSTATUS_PREGAME; +#if defined(GENERALS_ONLINE) + m_matchKeysRequired = FALSE; + m_matchStartFailed = FALSE; +#endif + QueryPerformanceFrequency((LARGE_INTEGER*)&m_perfCountFreq); m_nextFrameTime = 0; m_sawCRCMismatch = FALSE; @@ -389,6 +416,7 @@ void Network::SeedLatencyData(int highestLatency) m_conMgr->SeedLatencyData(highestLatency); } + #endif #if defined(GENERALS_ONLINE) @@ -545,6 +573,13 @@ Bool Network::processCommand(GameMessage* msg) if (TheGameLogic->getFrame() == 1) { #else if (TheGameLogic->getFrame() == 1) { +#endif +#if defined(GENERALS_ONLINE) + // Pregame updates drive loading; only entering gameplay requires completed setup. + if (!startPrerequisitesReady()) { + failMatchStart(); + return TRUE; + } #endif m_localStatus = NETLOCALSTATUS_INGAME; NetCommandList *netcmdlist = m_conMgr->getFrameCommandList(0); // clear out frame 0 since we skipped it @@ -759,7 +794,20 @@ void Network::update() } #endif +#if defined(GENERALS_ONLINE) + if (m_matchStartFailed) { + // Discard gameplay input, but leave the queued exit command for normal teardown. + GetCommandsFromCommandList(); + return; + } +#endif + GetCommandsFromCommandList(); // Remove commands from TheCommandList and send them to the connection manager. +#if defined(GENERALS_ONLINE) + if (m_matchStartFailed) { + return; + } +#endif if (m_conMgr != NULL) { if (m_localStatus == NETLOCALSTATUS_INGAME) { #if defined(GENERALS_ONLINE_RUN_FAST) @@ -777,10 +825,23 @@ void Network::update() } liteupdate(); +#if defined(GENERALS_ONLINE) + if (m_matchStartFailed) { + return; + } +#endif if (m_localStatus == NETLOCALSTATUS_LEFT) {// || (m_localStatus == NETLOCALSTATUS_LEAVING)) { endOfGameCheck(); +#if defined(GENERALS_ONLINE) + return; +#endif } +#if defined(GENERALS_ONLINE) + if (m_localStatus == NETLOCALSTATUS_POSTGAME) { + return; + } +#endif if (AllCommandsReady(TheGameLogic->getFrame())) { // If all the commands are ready for the next frame... m_conMgr->handleAllCommandsReady(); @@ -804,6 +865,11 @@ void Network::update() } void Network::liteupdate() { +#if defined(GENERALS_ONLINE) + if (m_matchStartFailed) { + return; + } +#endif #if defined(RTS_DEBUG) if (m_networkOn == FALSE) { @@ -887,7 +953,11 @@ Bool Network::timeForNewFrame() { * Returns true if the game commands for the next frame have been put on the command list. */ Bool Network::isFrameDataReady() { +#if defined(GENERALS_ONLINE) + return m_frameDataReady; +#else return (m_frameDataReady || (m_localStatus == NETLOCALSTATUS_LEFT)); +#endif } Bool Network::isStalling() @@ -1055,8 +1125,43 @@ void Network::updateLoadProgress(Int percent) } } +#if defined(GENERALS_ONLINE) +Bool Network::startPrerequisitesReady() const +{ + if (m_matchStartFailed) { + return FALSE; + } + if (m_matchKeysRequired) { + NetworkMesh* mesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + return mesh != nullptr && mesh->GetMatchKeys().IsReady(); + } + return TRUE; +} + +void Network::failMatchStart() +{ + if (m_matchStartFailed) { + return; + } + NetworkMesh* mesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + NetworkLog(ELogVerbosity::LOG_RELEASE, "[START] Missing match keys, leaving (keys=%zu/%zu)", + mesh != nullptr ? mesh->GetMatchKeys().GetPeerKeyCount() : 0, + mesh != nullptr ? mesh->GetMatchKeys().GetExpectedPeers() : 0); + m_matchStartFailed = TRUE; + quitGame(); + if (m_conMgr != nullptr) { + TheGameLogic->processProgressComplete((Int)m_conMgr->getLocalPlayerID()); + } +} +#endif + void Network::loadProgressComplete() { +#if defined(GENERALS_ONLINE) + if (m_matchStartFailed) { + return; + } +#endif if (m_conMgr != nullptr) { m_conMgr->loadProgressComplete(); } diff --git a/Core/GameEngine/Source/GameNetwork/NetworkUtil.cpp b/Core/GameEngine/Source/GameNetwork/NetworkUtil.cpp index 798e2c8055b..770f7072cb4 100644 --- a/Core/GameEngine/Source/GameNetwork/NetworkUtil.cpp +++ b/Core/GameEngine/Source/GameNetwork/NetworkUtil.cpp @@ -133,6 +133,8 @@ Bool DoesCommandRequireACommandID(NetCommandType type) case NETCOMMANDTYPE_DISCONNECTVOTE: case NETCOMMANDTYPE_DISCONNECTFRAME: case NETCOMMANDTYPE_DISCONNECTSCREENOFF: + case NETCOMMANDTYPE_SIGNEDFRAMEINFO: + case NETCOMMANDTYPE_SIGNATURES: return TRUE; default: return FALSE; @@ -222,6 +224,8 @@ const char* GetNetCommandTypeAsString(NetCommandType type) CASE_LABEL(NETCOMMANDTYPE_DISCONNECTFRAME) CASE_LABEL(NETCOMMANDTYPE_DISCONNECTSCREENOFF) CASE_LABEL(NETCOMMANDTYPE_DISCONNECTEND) + CASE_LABEL(NETCOMMANDTYPE_SIGNEDFRAMEINFO) + CASE_LABEL(NETCOMMANDTYPE_SIGNATURES) default: DEBUG_CRASH(("Unhandled NetCommandType in GetNetCommandTypeAsString")); return ""; diff --git a/Generals/Code/GameEngine/Source/Common/System/MemoryInit.cpp b/Generals/Code/GameEngine/Source/Common/System/MemoryInit.cpp index 4c941ae755a..cc47ff54ebc 100644 --- a/Generals/Code/GameEngine/Source/Common/System/MemoryInit.cpp +++ b/Generals/Code/GameEngine/Source/Common/System/MemoryInit.cpp @@ -565,6 +565,8 @@ static PoolSizeRec sizes[] = { "NetProgressCommandMsg", 32, 32 }, { "NetLoadCompleteCommandMsg", 32, 32 }, { "NetTimeOutGameStartCommandMsg", 32, 32 }, + { "NetSignedFrameCommandMsg", 32, 32 }, + { "NetSignaturesCommandMsg", 32, 32 }, { "NetWrapperCommandMsg", 32, 32 }, { "NetFileCommandMsg", 32, 32 }, { "NetFileAnnounceCommandMsg", 32, 32 }, diff --git a/GeneralsMD/Code/GameEngine/CMakeLists.txt b/GeneralsMD/Code/GameEngine/CMakeLists.txt index ee275be1383..626be25e2bb 100644 --- a/GeneralsMD/Code/GameEngine/CMakeLists.txt +++ b/GeneralsMD/Code/GameEngine/CMakeLists.txt @@ -1141,6 +1141,7 @@ set(GAMEENGINE_SRC Include/GameNetwork/GeneralsOnline/Vendor/DiscordRPC/discord_rpc.h Include/GameNetwork/GeneralsOnline/NetworkBitstream.h Include/GameNetwork/GeneralsOnline/ConnectionRating.h + Include/GameNetwork/GeneralsOnline/MatchKeys.h Include/GameNetwork/GeneralsOnline/NetworkMesh.h Include/GameNetwork/GeneralsOnline/NetBytes.h Include/GameNetwork/GeneralsOnline/PacketFeedback.h @@ -1170,6 +1171,7 @@ set(GAMEENGINE_SRC Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp Source/GameNetwork/GeneralsOnline/ConnectionRating.cpp + Source/GameNetwork/GeneralsOnline/MatchKeys.cpp Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp Source/GameNetwork/GeneralsOnline/PacketFeedback.cpp Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp diff --git a/GeneralsMD/Code/GameEngine/Include/GameLogic/GameLogic.h b/GeneralsMD/Code/GameEngine/Include/GameLogic/GameLogic.h index c3a66f6c326..51c3b6c9e30 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameLogic/GameLogic.h +++ b/GeneralsMD/Code/GameEngine/Include/GameLogic/GameLogic.h @@ -223,6 +223,9 @@ class GameLogic : public SubsystemInterface, public Snapshot UnsignedInt getFrameObjectsChangedTriggerAreas() {return m_frameObjectsChangedTriggerAreas;} void exitGame(); +#if defined(GENERALS_ONLINE) + void processPendingGameExit(); +#endif void quit(Bool toDesktop); void clearGameData(Bool showScoreScreen = TRUE); ///< Clear the game data void closeWindows(); diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/MatchKeys.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/MatchKeys.h new file mode 100644 index 00000000000..48666f5cae3 --- /dev/null +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/MatchKeys.h @@ -0,0 +1,77 @@ +#pragma once + +#include +#include +#include +#include +#include + +typedef struct evp_pkey_st EVP_PKEY; + +// Ed25519 keys: each client makes a key pair when it joins or creates a lobby and announces the public key over each peer's own connection, +// a key counts for the user whose connection it came on. A match binds to the keys when it starts: the match id and the roster go into everything +// that is signed, so a signature is good for that match only. See ConnectionManager for what is signed. +// +// Channel messages, reliable: +// announcement: u8 kind (2), i64 user id, 32 bytes public key +// status: u8 kind (3), u8 count, count x i64 user id of the peers whose key the sender holds +class MatchKeys +{ +public: + enum { PUBLIC_KEY_SIZE = 32, SIGNATURE_SIZE = 64, ANNOUNCEMENT_SIZE = 1 + 8 + PUBLIC_KEY_SIZE, MAX_STATUS_USERS = 16 }; + enum { KIND_ANNOUNCEMENT = 2, KIND_STATUS = 3 }; + + MatchKeys(); + ~MatchKeys(); + + bool BeginLobby(int64_t localUserID); + /// Drops the key pair and every peer key, call when the lobby is left. + void End(); + bool HasLocalKey() const { return m_privateKey != nullptr; } + int64_t GetLocalUserID() const { return m_localUserID; } + + bool BeginMatch(uint64_t matchID, const std::set& expectedPeers); + void EndMatch(); + bool IsActive() const { return m_active; } + + std::vector BuildAnnouncement() const; + /// The keys held, the host reads it to see that every pair has exchanged keys. + std::vector BuildStatus() const; + + /// Takes an announcement that arrived on the connection of connectionUserID. False and a reason when rejected, changed when a key was added or replaced. + bool OnAnnouncement(int64_t connectionUserID, const uint8_t* data, size_t len, const char*& reason, bool& changed); + /// Takes a status that arrived on the connection of connectionUserID. + bool OnStatus(int64_t connectionUserID, const uint8_t* data, size_t len, const char*& reason, bool& changed); + + /// Drops the key of a peer that left the lobby, never one of a bound match roster. + bool DropPeer(int64_t userID); + + size_t GetPeerKeyCount() const { return m_peerKeys.size(); } + size_t GetExpectedPeers() const { return m_expectedPeers.size(); } + /// True when bound and a key is held for every peer of the roster. + bool IsReady() const; + bool ExpectsPeer(int64_t userID) const { return m_expectedPeers.count(userID) != 0; } + bool HasPeerKey(int64_t userID) const { return m_peerKeys.find(userID) != m_peerKeys.end(); } + std::vector GetPeerKeyUsers() const; + /// True when peer reported holding the keys of every user in needed, peer itself excepted. + bool PeerHoldsAll(int64_t peer, const std::set& needed) const; + uint64_t GetMatchID() const { return m_matchID; } + + bool Sign(const uint8_t* data, size_t len, uint8_t* signature) const; + /// Checks a signature of userID over data, false without a key of that user or when not bound to a match. + bool Verify(int64_t userID, const uint8_t* data, size_t len, const uint8_t* signature); + +private: + void FreePeerKeyObject(int64_t userID); + void FreePeerKeyObjects(); + + bool m_active; + uint64_t m_matchID; + int64_t m_localUserID; + std::set m_expectedPeers; + EVP_PKEY* m_privateKey; + std::array m_publicKey; + std::map> m_peerKeys; + std::map> m_peerHolds; ///< Per peer, the users whose key it said it holds. + std::map m_peerKeyObjects; ///< The peer keys parsed for OpenSSL, made on first use. +}; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMPGame.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMPGame.h index 1275b1bc5c8..11281e7cd92 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMPGame.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMPGame.h @@ -84,7 +84,7 @@ class NGMPGame : public GameInfo for (Int slotNum = 0; slotNum < MAX_SLOTS; ++slotNum) { const NGMPGameSlot* pSlot = (const NGMPGameSlot*)getConstSlot(slotNum); - if (pSlot != nullptr && pSlot->m_userID == userID) + if (pSlot != nullptr && pSlot->isHuman() && pSlot->m_userID == userID) { return slotNum; } @@ -92,6 +92,12 @@ class NGMPGame : public GameInfo return -1; } + int64_t getUserOfSlot(Int slotNum) const + { + const NGMPGameSlot* pSlot = (slotNum >= 0 && slotNum < MAX_SLOTS) ? (const NGMPGameSlot*)getConstSlot(slotNum) : nullptr; + return (pSlot != nullptr && pSlot->isHuman()) ? pSlot->m_userID : -1; + } + bool HasCommittedOutcome() const { return m_bHasCommittedOutcome; } void SetHasCommittedOutcome() { m_bHasCommittedOutcome = true; } diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h index b22c0ced24e..e439210828b 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h @@ -11,7 +11,8 @@ enum class ENetworkChannel : BYTE { NETWORK_CHANNEL_GAME = 0, NETWORK_CHANNEL_AC = 1, - NETWORK_CHANNEL_PING = 2 + NETWORK_CHANNEL_PING = 2, + NETWORK_CHANNEL_KEYS = 3 // match signing keys, sent reliably }; #define CHECK_MAIN_THREAD assert(std::this_thread::get_id() == NGMP_OnlineServicesManager::g_MainThreadID); diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetBytes.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetBytes.h index c173e617a29..f7e89df57b4 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetBytes.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetBytes.h @@ -14,3 +14,21 @@ inline uint16_t ReadU16(const uint8_t* in) { return static_cast(in[0] | (in[1] << 8)); } + +inline void WriteU64(uint8_t* out, uint64_t value) +{ + for (int i = 0; i < 8; ++i) + { + out[i] = static_cast(value >> (8 * i)); + } +} + +inline uint64_t ReadU64(const uint8_t* in) +{ + uint64_t value = 0; + for (int i = 0; i < 8; ++i) + { + value |= static_cast(in[i]) << (8 * i); + } + return value; +} diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h index 3faae1def69..489c97b6dfe 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h @@ -10,6 +10,7 @@ #include "GameNetwork/ConnectionStats.h" #include "GameNetwork/GeneralsOnline/ConnectionRating.h" #include "GameNetwork/GeneralsOnline/PacketFeedback.h" +#include "GameNetwork/GeneralsOnline/MatchKeys.h" class NetRoom_ChatMessagePacket; @@ -119,6 +120,9 @@ class PlayerConnection int SendEnveloped(ENetworkChannel channel, const uint8_t* payload, size_t len, int sendFlags); ///< returns an EResult bool GetOutgoingLoss(int windowMs, PacketFeedback::Summary& summary); ///< our packets to the peer, false without feedback bool GetIncomingLoss(int windowMs, PacketFeedback::Summary& summary); ///< the peer's packets to us + + void HandleKeyMessage(const uint8_t* data, size_t len); ///< A match key announcement arrived on this connection. + int GetJitter(); int ComputeConnectionRating(); ///< 1..5 bars, -1 while it is not known yet; cached for 250 ms. int UpdateConnectionRating(); @@ -202,6 +206,8 @@ class NetworkMesh void RegisterConnectivity(int64_t userID); void UpdateConnectivity(PlayerConnection* connection); + MatchKeys m_matchKeys; + std::function m_cbOnConnected = nullptr; void RegisterForConnectionEvents(std::function cb) { @@ -229,6 +235,23 @@ class NetworkMesh return highestLatency; } + // The signing keys, see MatchKeys.h + bool BeginLobbyKeys(); ///< Makes this client's key, call when it joins or creates a lobby. + void TickLobbyKeys(int64_t nowMs); ///< Announces the key to the peers that lack it and tells them which keys we hold. + bool BeginMatchKeys(); ///< Binds the keys to the match that is starting, false when a key of the roster is missing. + void EndMatchKeys(); ///< Unbinds from the match, the lobby keys stay. + void OnKeyAnnouncement(int64_t connectionUserID, const uint8_t* data, size_t len); + + /// A lobby member the host has to wait for before it starts. + struct LobbyStartBlocker + { + int64_t userID; + bool notConnected; ///< Else the connection is up but the keys are not exchanged with everyone yet. + }; + void GetLobbyStartBlockers(std::vector& blockers); + const MatchKeys& GetMatchKeys() const { return m_matchKeys; } + MatchKeys& GetMatchKeys() { return m_matchKeys; } + Real getMaximumHistoricalLatency() { int highestLatency = 0; @@ -304,6 +327,14 @@ class NetworkMesh bool m_bDisconnected = false; + struct KeySync + { + int64_t nextAnnounceMs = 0; + int64_t nextStatusMs = 0; + std::vector lastStatus; + }; + std::map m_keySync; ///< Per peer, when the key and the held keys were last sent. + std::string m_strTurnUsername; std::string m_strTurnToken; std::string m_strTurnUsernameString; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index 51205b64f77..3eb1c9dbb42 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -205,8 +205,16 @@ class WebSocket // Queues the message; Tick() flushes it on the main thread. Thread-safe. void Send(const char* message); + // An inbound P2P signal and the user the service says it came from. The service sets the sender itself + // from the authenticated websocket session, the payload is whatever the peer wrote. + struct PendingSignal + { + int64_t senderUserID = -1; + std::vector payload; + }; + // Thread-safe queue of inbound P2P signal payloads; drained by CSignalingClient::Poll(). - void PushPendingSignal(std::vector payload) + void PushPendingSignal(int64_t senderUserID, std::vector payload) { std::scoped_lock lock(m_pendingSignalsMutex); @@ -217,13 +225,16 @@ class WebSocket m_pendingSignals.pop(); } - m_pendingSignals.push(std::move(payload)); + PendingSignal signal; + signal.senderUserID = senderUserID; + signal.payload = std::move(payload); + m_pendingSignals.push(std::move(signal)); } - std::queue> DrainPendingSignals() + std::queue DrainPendingSignals() { std::scoped_lock lock(m_pendingSignalsMutex); - std::queue> drained; + std::queue drained; drained.swap(m_pendingSignals); return drained; } @@ -267,7 +278,7 @@ class WebSocket std::vector m_vecQueuedOutboungMsgs; std::mutex m_pendingSignalsMutex; - std::queue> m_pendingSignals; + std::queue m_pendingSignals; }; enum class ERoomFlags : int diff --git a/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp b/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp index 0c497745de0..863783c3bcc 100644 --- a/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp +++ b/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp @@ -1023,6 +1023,14 @@ void GameEngine::update() } } +#if defined(GENERALS_ONLINE) + if (TheNetwork != nullptr && TheNetwork->isGameExitPending()) { + // Teardown can delete Network and must not run another gameplay update. + TheGameLogic->processPendingGameExit(); + return; + } +#endif + // TheSuperHackers @info Ignores frozen time because the script engine needs updating in the logic update regardless. if (canUpdateGameLogic(FramePacer::IgnoreFrozenTime)) { diff --git a/GeneralsMD/Code/GameEngine/Source/Common/System/MemoryInit.cpp b/GeneralsMD/Code/GameEngine/Source/Common/System/MemoryInit.cpp index 2881c795981..e082fe22f4d 100644 --- a/GeneralsMD/Code/GameEngine/Source/Common/System/MemoryInit.cpp +++ b/GeneralsMD/Code/GameEngine/Source/Common/System/MemoryInit.cpp @@ -613,6 +613,8 @@ static PoolSizeRec sizes[] = { "NetProgressCommandMsg", 32, 32 }, { "NetLoadCompleteCommandMsg", 32, 32 }, { "NetTimeOutGameStartCommandMsg", 32, 32 }, + { "NetSignedFrameCommandMsg", 32, 32 }, + { "NetSignaturesCommandMsg", 32, 32 }, { "NetWrapperCommandMsg", 32, 32 }, { "NetFileCommandMsg", 32, 32 }, { "NetFileAnnounceCommandMsg", 32, 32 }, diff --git a/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp b/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp index b04c4c1180d..84890487da5 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp @@ -982,6 +982,29 @@ static void WOLLockSettings() } } +// The host does not start while a member is not connected or has not exchanged keys with everyone. +static Bool WOLLobbyReadyToStart(void) +{ + NGMP_OnlineServices_LobbyInterface* pLobbyInterface = NGMP_OnlineServicesManager::GetInterface(); + NetworkMesh* pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + if (pLobbyInterface == nullptr || pMesh == nullptr) + { + return TRUE; + } + + std::vector blockers; + pMesh->GetLobbyStartBlockers(blockers); + for (const NetworkMesh::LobbyStartBlocker& blocker : blockers) + { + const LobbyMemberEntry member = pLobbyInterface->GetRoomMemberFromID(blocker.userID); + UnicodeString text; + text.format(blocker.notConnected ? L"Connections: Waiting for %s to connect." : L"Connections: Waiting for %s to exchange keys.", from_utf8(member.display_name).c_str()); + GadgetListBoxAddEntryText(listboxGameSetupChat, text, GameMakeColor(255, 194, 15, 255), -1, -1); + NetworkLog(ELogVerbosity::LOG_RELEASE, "[START] Not starting, waiting for user %lld to %s", static_cast(blocker.userID), blocker.notConnected ? "connect" : "exchange keys"); + } + return blockers.empty(); +} + static void StartPressed() { Bool isReady = TRUE; @@ -1044,6 +1067,11 @@ static void StartPressed() return; } + if (!WOLLobbyReadyToStart()) + { + return; + } + // see if everyone's accepted and count the number of players in the game UnicodeString mapDisplayName; const MapMetaData *mapData = TheMapCache->findMap( myGame->getMap() ); @@ -2600,15 +2628,20 @@ void WOLGameSetupMenuUpdate( WindowLayout * layout, void *userData) // are we done? if (secondsRemaining <= 0) { - s_matchStartCountdownWasRunning = false; // stop countdown TheNGMPGame->StopCountdown(); - // send start game packet - std::shared_ptr pWS = NGMP_OnlineServicesManager::GetWebSocket(); - if (pWS != nullptr) + // a member that joined during the countdown is not ready yet, the buttons come back with the stopped countdown + if (WOLLobbyReadyToStart()) { - pWS->SendData_StartGame(); + s_matchStartCountdownWasRunning = false; + + // send start game packet + std::shared_ptr pWS = NGMP_OnlineServicesManager::GetWebSocket(); + if (pWS != nullptr) + { + pWS->SendData_StartGame(); + } } } } diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/MatchKeys.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/MatchKeys.cpp new file mode 100644 index 00000000000..767a12adac5 --- /dev/null +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/MatchKeys.cpp @@ -0,0 +1,333 @@ +#include "GameNetwork/GeneralsOnline/MatchKeys.h" +#include "GameNetwork/GeneralsOnline/NetBytes.h" + +#include + +#include + +MatchKeys::MatchKeys() + : m_active(false) + , m_matchID(0) + , m_localUserID(-1) + , m_privateKey(nullptr) +{ + m_publicKey.fill(0); +} + +MatchKeys::~MatchKeys() +{ + End(); +} + +bool MatchKeys::BeginLobby(int64_t localUserID) +{ + if (m_privateKey != nullptr && m_localUserID == localUserID) + { + return true; + } + End(); + + EVP_PKEY_CTX* ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_ED25519, nullptr); + if (ctx == nullptr) + { + return false; + } + + EVP_PKEY* key = nullptr; + const bool generated = EVP_PKEY_keygen_init(ctx) == 1 && EVP_PKEY_keygen(ctx, &key) == 1; + EVP_PKEY_CTX_free(ctx); + if (!generated || key == nullptr) + { + return false; + } + + size_t publicKeySize = PUBLIC_KEY_SIZE; + if (EVP_PKEY_get_raw_public_key(key, m_publicKey.data(), &publicKeySize) != 1 || publicKeySize != PUBLIC_KEY_SIZE) + { + EVP_PKEY_free(key); + return false; + } + + m_privateKey = key; + m_localUserID = localUserID; + return true; +} + +void MatchKeys::End() +{ + if (m_privateKey != nullptr) + { + EVP_PKEY_free(m_privateKey); + m_privateKey = nullptr; + } + m_publicKey.fill(0); + FreePeerKeyObjects(); + m_peerKeys.clear(); + m_peerHolds.clear(); + m_active = false; + m_expectedPeers.clear(); + m_matchID = 0; + m_localUserID = -1; +} + +bool MatchKeys::BeginMatch(uint64_t matchID, const std::set& expectedPeers) +{ + if (m_privateKey == nullptr) + { + return false; + } + m_matchID = matchID; + m_expectedPeers = expectedPeers; + m_active = true; + return true; +} + +void MatchKeys::EndMatch() +{ + m_active = false; + m_expectedPeers.clear(); + m_matchID = 0; +} + +bool MatchKeys::IsReady() const +{ + if (!m_active) + { + return false; + } + for (int64_t userID : m_expectedPeers) + { + if (!HasPeerKey(userID)) + { + return false; + } + } + return true; +} + +std::vector MatchKeys::GetPeerKeyUsers() const +{ + std::vector users; + for (const auto& entry : m_peerKeys) + { + users.push_back(entry.first); + } + return users; +} + +bool MatchKeys::PeerHoldsAll(int64_t peer, const std::set& needed) const +{ + auto holds = m_peerHolds.find(peer); + if (holds == m_peerHolds.end()) + { + return false; + } + for (int64_t userID : needed) + { + if (userID != peer && holds->second.count(userID) == 0) + { + return false; + } + } + return true; +} + +bool MatchKeys::DropPeer(int64_t userID) +{ + if (m_active && ExpectsPeer(userID)) + { + return false; + } + FreePeerKeyObject(userID); + m_peerHolds.erase(userID); + return m_peerKeys.erase(userID) != 0; +} + +void MatchKeys::FreePeerKeyObject(int64_t userID) +{ + auto parsed = m_peerKeyObjects.find(userID); + if (parsed != m_peerKeyObjects.end()) + { + EVP_PKEY_free(parsed->second); + m_peerKeyObjects.erase(parsed); + } +} + +void MatchKeys::FreePeerKeyObjects() +{ + for (auto& entry : m_peerKeyObjects) + { + EVP_PKEY_free(entry.second); + } + m_peerKeyObjects.clear(); +} + +bool MatchKeys::Sign(const uint8_t* data, size_t len, uint8_t* signature) const +{ + if (!m_active || m_privateKey == nullptr) + { + return false; + } + + EVP_MD_CTX* ctx = EVP_MD_CTX_new(); + if (ctx == nullptr) + { + return false; + } + + size_t signatureSize = SIGNATURE_SIZE; + const bool signed_ = EVP_DigestSignInit(ctx, nullptr, nullptr, nullptr, m_privateKey) == 1 + && EVP_DigestSign(ctx, signature, &signatureSize, data, len) == 1 && signatureSize == SIGNATURE_SIZE; + EVP_MD_CTX_free(ctx); + return signed_; +} + +bool MatchKeys::Verify(int64_t userID, const uint8_t* data, size_t len, const uint8_t* signature) +{ + if (!m_active) + { + return false; + } + + EVP_PKEY* key = nullptr; + auto parsed = m_peerKeyObjects.find(userID); + if (parsed != m_peerKeyObjects.end()) + { + key = parsed->second; + } + else + { + auto raw = m_peerKeys.find(userID); + if (raw == m_peerKeys.end()) + { + return false; + } + key = EVP_PKEY_new_raw_public_key(EVP_PKEY_ED25519, nullptr, raw->second.data(), PUBLIC_KEY_SIZE); + if (key == nullptr) + { + return false; + } + m_peerKeyObjects[userID] = key; + } + + EVP_MD_CTX* ctx = EVP_MD_CTX_new(); + if (ctx == nullptr) + { + return false; + } + + const bool valid = EVP_DigestVerifyInit(ctx, nullptr, nullptr, nullptr, key) == 1 + && EVP_DigestVerify(ctx, signature, SIGNATURE_SIZE, data, len) == 1; + EVP_MD_CTX_free(ctx); + return valid; +} + +std::vector MatchKeys::BuildAnnouncement() const +{ + std::vector message(ANNOUNCEMENT_SIZE); + message[0] = KIND_ANNOUNCEMENT; + WriteU64(message.data() + 1, static_cast(m_localUserID)); + memcpy(message.data() + 9, m_publicKey.data(), PUBLIC_KEY_SIZE); + return message; +} + +std::vector MatchKeys::BuildStatus() const +{ + std::vector message; + message.push_back(KIND_STATUS); + message.push_back(0); + uint8_t count = 0; + for (const auto& entry : m_peerKeys) + { + if (count >= MAX_STATUS_USERS) + { + break; + } + uint8_t id[8]; + WriteU64(id, static_cast(entry.first)); + message.insert(message.end(), id, id + 8); + ++count; + } + message[1] = count; + return message; +} + +bool MatchKeys::OnAnnouncement(int64_t connectionUserID, const uint8_t* data, size_t len, const char*& reason, bool& changed) +{ + changed = false; + if (data == nullptr || len != ANNOUNCEMENT_SIZE || data[0] != KIND_ANNOUNCEMENT) + { + reason = "malformed or another version"; + return false; + } + + const int64_t announcedUserID = static_cast(ReadU64(data + 1)); + if (announcedUserID != connectionUserID) + { + reason = "it names another user than the connection it arrived on"; + return false; + } + if (m_privateKey == nullptr || connectionUserID == m_localUserID) + { + reason = "this client has no key of its own yet or the key is its own"; + return false; + } + if (m_active && !ExpectsPeer(connectionUserID)) + { + reason = "the user is not in the match roster"; + return false; + } + + std::array key; + memcpy(key.data(), data + 9, PUBLIC_KEY_SIZE); + + auto existing = m_peerKeys.find(connectionUserID); + if (existing != m_peerKeys.end()) + { + if (existing->second == key) + { + reason = nullptr; + return true; // the same key again + } + if (m_active) + { + reason = "this user already announced a different key"; + return false; + } + // a client that came back into the lobby makes a new key, the connection it arrives on vouches for it + FreePeerKeyObject(connectionUserID); + m_peerHolds.erase(connectionUserID); + } + + m_peerKeys[connectionUserID] = key; + changed = true; + reason = nullptr; + return true; +} + +bool MatchKeys::OnStatus(int64_t connectionUserID, const uint8_t* data, size_t len, const char*& reason, bool& changed) +{ + changed = false; + if (data == nullptr || len < 2 || data[0] != KIND_STATUS) + { + reason = "malformed or another version"; + return false; + } + const size_t count = data[1]; + if (count > MAX_STATUS_USERS || len != 2 + count * 8) + { + reason = "malformed status"; + return false; + } + + std::set holds; + for (size_t i = 0; i < count; ++i) + { + holds.insert(static_cast(ReadU64(data + 2 + i * 8))); + } + auto existing = m_peerHolds.find(connectionUserID); + changed = existing == m_peerHolds.end() || existing->second != holds; + m_peerHolds[connectionUserID] = holds; + reason = nullptr; + return true; +} diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp index 490bfcf0e33..bf00a650e6c 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp @@ -452,12 +452,21 @@ void NGMPGame::launchGame(void) TheNetwork = NetworkInterface::createNetwork(); TheNetwork->init(); + TheNetwork->requireMatchKeys(); NetworkMesh* pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); - if (pMesh != nullptr) + if (pMesh == nullptr || !pMesh->BeginMatchKeys()) { - - TheNetwork->SeedLatencyData(pMesh->getMaximumHistoricalLatency()); + delete TheNetwork; + TheNetwork = nullptr; + setGameInProgress(FALSE); + UnicodeString error; + error.translate("Could not establish authenticated game connections. This transport may not support player authentication."); + GSMessageBoxOk(TheGameText->fetch("GUI:Error"), error); + void PopBackToLobby(void); + PopBackToLobby(); + return; } + TheNetwork->SeedLatencyData(pMesh->getMaximumHistoricalLatency()); // TODO_NGMP: Do we really care about these values anymore TheNetwork->setLocalAddress(getLocalIP(), 8888); diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp index b6fca95e0f9..f42004617eb 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp @@ -4,6 +4,8 @@ #include "GameNetwork/GeneralsOnline/NGMP_interfaces.h" #include +#include +#include #include "GameNetwork/NetworkDefs.h" #include "GameNetwork/NetworkInterface.h" #include "Common/GlobalData.h" @@ -48,6 +50,27 @@ static void CleanupPendingConnSignalingDeletions() } } +// The user id of a handshake identity, which is plain text chosen by the peer. +static bool ParseUserIdentity(const SteamNetworkingIdentity& identity, int64_t& userId) +{ + const char* szIdentity = identity.GetGenericString(); + if (szIdentity == nullptr) + { + return false; + } + + const size_t identityLen = strlen(szIdentity); + const char* szDigits = (szIdentity[0] == '-') ? szIdentity + 1 : szIdentity; + const size_t numDigits = identityLen - (szDigits - szIdentity); + if (numDigits == 0 || numDigits > 19 || strspn(szDigits, "0123456789") != numDigits) + { + return false; + } + + const std::from_chars_result parseResult = std::from_chars(szIdentity, szIdentity + identityLen, userId); + return parseResult.ec == std::errc() && parseResult.ptr == szIdentity + identityLen; +} + // Called on connection state transitions. Always runs on the main thread via RunCallbacks(), so // m_mapConnections is accessed here without m_mapConnectionsMutex. void OnSteamNetConnectionStatusChanged(SteamNetConnectionStatusChangedCallback_t* pInfo) @@ -234,6 +257,15 @@ void OnSteamNetConnectionStatusChanged(SteamNetConnectionStatusChangedCallback_t assert(plrConnection.m_hSteamConnection == k_HSteamNetConnection_Invalid); // not really a bug in this code, but a bug in the test #endif + // the identity of the peer must be the user this connection is for + int64_t remoteUserID = 0; + if (!ParseUserIdentity(pInfo->m_info.m_identityRemote, remoteUserID) || remoteUserID != plrConnection.m_userID) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] Rejecting incoming connection %s: its identity is not user %lld", pInfo->m_info.m_szConnectionDescription, static_cast(plrConnection.m_userID)); + SteamNetworkingSockets()->CloseConnection(pInfo->m_hConn, 1001, "Identity mismatch (Rejected)", false); + return; + } + if (pInfo != nullptr) { @@ -362,6 +394,15 @@ void OnSteamNetConnectionStatusChanged(SteamNetConnectionStatusChangedCallback_t { PlayerConnection& plrConnection = connections[connectionID]; + // the peer that finished the handshake must be the user this connection was made for + int64_t remoteUserID = 0; + if (!ParseUserIdentity(pInfo->m_info.m_identityRemote, remoteUserID) || remoteUserID != plrConnection.m_userID) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] Closing connection %s: its identity is not user %lld", pInfo->m_info.m_szConnectionDescription, static_cast(plrConnection.m_userID)); + SteamNetworkingSockets()->CloseConnection(pInfo->m_hConn, 1001, "Identity mismatch (Rejected)", false); + break; + } + plrConnection.UpdateState(EConnectionState::CONNECTED_DIRECT, pMesh); } @@ -486,9 +527,17 @@ class CSignalingClient : public ISignalingClient // identity does not have spaces, since our protocol doesn't permit it. NetworkLog(ELogVerbosity::LOG_DEBUG, "Creating signaling session for peer '%s'\n", sIdentityPeer.c_str()); - // Silence warnings - (void)errMsg; - int64_t user_id = std::stoll(identityPeer.GetGenericString()); + // The identity comes from the remote peer, so only accept a plain integer user id + int64_t user_id = 0; + const bool bValidIdentity = ParseUserIdentity(identityPeer, user_id); + + if (!bValidIdentity) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SIGNAL] Rejecting connection, peer identity '%s' is not a valid user id", sIdentityPeer.c_str()); + snprintf(errMsg, sizeof(errMsg), "Invalid peer identity"); + return nullptr; + } + return new ConnectionSignaling(this, user_id); } @@ -522,7 +571,7 @@ class CSignalingClient : public ISignalingClient sendBatch.pop_front(); } - std::queue> pendingSignals = pWS->DrainPendingSignals(); + std::queue pendingSignals = pWS->DrainPendingSignals(); // Now dispatch any buffered signals if (!pendingSignals.empty()) @@ -533,13 +582,15 @@ class CSignalingClient : public ISignalingClient // NOTE: outbound msg doesnt need sender ID, we only need that to determine target on the server, everything else is included in the payload // // Get the next signal - std::vector signalData = pendingSignals.front(); + const WebSocket::PendingSignal pendingSignal = pendingSignals.front(); + const std::vector& signalData = pendingSignal.payload; pendingSignals.pop(); // Setup a context object that can respond if this signal is a connection request. struct Context : ISteamNetworkingSignalingRecvContext { CSignalingClient* m_pOwner; + int64_t m_serverSender = -1; // the user the service delivered this signal from virtual ISteamNetworkingConnectionSignaling* OnConnectRequest( HSteamNetConnection hConn, @@ -557,6 +608,16 @@ class CSignalingClient : public ISignalingClient // Also, note that if there was routing/session info, it should have been in // our envelope that we know how to parse, and we should save it off in this // context object. + // The identity in the handshake is whatever the peer wrote. The service stamped the signal with + // the user it came from, a request claiming to be somebody else is not answered. + int64_t claimedUserID = 0; + if (ParseUserIdentity(identityPeer, claimedUserID) && claimedUserID != m_serverSender) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] Rejecting connection request: the handshake claims user %lld but the service delivered the signal from user %lld", + static_cast(claimedUserID), static_cast(m_serverSender)); + return nullptr; + } + SteamNetworkingErrMsg ignoreErrMsg; return m_pOwner->CreateSignalingForConnection(identityPeer, ignoreErrMsg); } @@ -581,6 +642,7 @@ class CSignalingClient : public ISignalingClient }; Context context; context.m_pOwner = this; + context.m_serverSender = pendingSignal.senderUserID; // Dispatch. // Remember: From inside this function, our context object might get callbacks. @@ -773,6 +835,9 @@ NetworkMesh::NetworkMesh() } m_bInitialized = true; + + // the key of this client for the lobby, peers get it as they connect + BeginLobbyKeys(); } @@ -1027,6 +1092,252 @@ void NetworkMesh::StartConnectionSignalling(const char* szMiddlewareID, int64_t } +bool NetworkMesh::BeginLobbyKeys() +{ + NGMP_OnlineServices_AuthInterface* auth = NGMP_OnlineServicesManager::GetInterface(); + if (auth == nullptr || AnticheatPlugInterface::DoesACPluginProvideSecureGameTransport()) + { + return false; + } + + std::lock_guard lock(m_mapConnectionsMutex); + if (!m_matchKeys.BeginLobby(auth->GetUserID())) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[KEYS] Could not make the key pair of this client"); + return false; + } + m_keySync.clear(); + return true; +} + +void NetworkMesh::TickLobbyKeys(int64_t nowMs) +{ + // once a match is bound its keys are done, the roster does not change + if (!m_matchKeys.HasLocalKey() || m_matchKeys.IsActive()) + { + return; + } + + std::lock_guard lock(m_mapConnectionsMutex); + + // the key of a peer that is gone is dropped, it announces again when it is back + for (int64_t userID : m_matchKeys.GetPeerKeyUsers()) + { + auto it = m_mapConnections.find(userID); + if (it == m_mapConnections.end() || !it->second.IsValid()) + { + m_matchKeys.DropPeer(userID); + m_keySync.erase(userID); + NetworkLog(ELogVerbosity::LOG_RELEASE, "[KEYS] Dropped the key of user %lld, it is no longer connected", static_cast(userID)); + } + } + + const std::set self = { m_matchKeys.GetLocalUserID() }; + const std::vector announcement = m_matchKeys.BuildAnnouncement(); + const std::vector status = m_matchKeys.BuildStatus(); + for (auto& entry : m_mapConnections) + { + PlayerConnection& connection = entry.second; + if (connection.GetState() != EConnectionState::CONNECTED_DIRECT || connection.m_hSteamConnection == k_HSteamNetConnection_Invalid) + { + m_keySync.erase(entry.first); + continue; + } + + KeySync& sync = m_keySync[entry.first]; + const bool announce = nowMs >= sync.nextAnnounceMs && !m_matchKeys.PeerHoldsAll(entry.first, self); + const bool report = nowMs >= sync.nextStatusMs || status != sync.lastStatus; + if (!announce && !report) + { + continue; + } + + for (int pass = 0; pass < 2; ++pass) + { + if (pass == 0 ? !announce : !report) + { + continue; + } + const std::vector& body = pass == 0 ? announcement : status; + std::vector packet(sizeof(ENetworkChannel) + body.size()); + packet[0] = static_cast(ENetworkChannel::NETWORK_CHANNEL_KEYS); + memcpy(packet.data() + sizeof(ENetworkChannel), body.data(), body.size()); + const EResult result = SteamNetworkingSockets()->SendMessageToConnection(connection.m_hSteamConnection, + packet.data(), static_cast(packet.size()), k_nSteamNetworkingSend_ReliableNoNagle, nullptr); + if (result != k_EResultOK) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[KEYS] Could not send to user %lld, error %d", static_cast(entry.first), static_cast(result)); + } + } + if (announce) + { + sync.nextAnnounceMs = nowMs + 1000; + } + if (report) + { + sync.nextStatusMs = nowMs + 5000; + sync.lastStatus = status; + } + } +} + +bool NetworkMesh::BeginMatchKeys() +{ + NGMP_OnlineServices_AuthInterface* auth = NGMP_OnlineServicesManager::GetInterface(); + NGMP_OnlineServices_LobbyInterface* lobby = NGMP_OnlineServicesManager::GetInterface(); + if (auth == nullptr || lobby == nullptr || TheNGMPGame == nullptr) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[KEYS] Cannot authenticate match without auth, lobby and roster"); + EndMatchKeys(); + return false; + } + + std::set peers; + bool haveLocalPlayer = false; + for (Int i = 0; i < MAX_SLOTS; ++i) + { + const NGMPGameSlot* slot = (const NGMPGameSlot*)TheNGMPGame->getConstSlot(i); + if (slot == nullptr || !slot->isHuman()) + { + continue; + } + if (slot->m_userID < 0 || !peers.insert(slot->m_userID).second) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[KEYS] Invalid or duplicate user identity in match roster"); + EndMatchKeys(); + return false; + } + haveLocalPlayer |= slot->m_userID == auth->GetUserID(); + } + if (!haveLocalPlayer) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[KEYS] Local user is missing from match roster"); + EndMatchKeys(); + return false; + } + peers.erase(auth->GetUserID()); + if (!peers.empty() && AnticheatPlugInterface::DoesACPluginProvideSecureGameTransport()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[KEYS] Cannot authenticate peers on plugin transport without sender attribution"); + EndMatchKeys(); + return false; + } + + std::lock_guard lock(m_mapConnectionsMutex); + if (!m_matchKeys.HasLocalKey() && !BeginLobbyKeys()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[KEYS] Could not make the key pair of this client"); + return false; + } + + // the keys came in while the lobby was open, a launch never waits for one + for (int64_t userID : peers) + { + if (!m_matchKeys.HasPeerKey(userID)) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[KEYS] No key of user %lld in the roster of the match", static_cast(userID)); + EndMatchKeys(); + return false; + } + } + + if (!m_matchKeys.BeginMatch(lobby->GetCurrentMatchID(), peers)) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[KEYS] Could not bind the keys to the match"); + return false; + } + NetworkLog(ELogVerbosity::LOG_RELEASE, "[KEYS] Keys bound to match %llu with %zu peers", static_cast(lobby->GetCurrentMatchID()), peers.size()); + return true; +} + +void NetworkMesh::EndMatchKeys() +{ + m_matchKeys.EndMatch(); +} + +void NetworkMesh::GetLobbyStartBlockers(std::vector& blockers) +{ + blockers.clear(); + NGMP_OnlineServices_AuthInterface* auth = NGMP_OnlineServicesManager::GetInterface(); + NGMP_OnlineServices_LobbyInterface* lobby = NGMP_OnlineServicesManager::GetInterface(); + if (auth == nullptr || lobby == nullptr) + { + return; + } + + std::set humans; + for (const LobbyMemberEntry& member : lobby->GetCurrentLobby().members) + { + if (member.IsHuman()) + { + humans.insert(member.user_id); + } + } + + // on a plugin transport there are no keys, the launch says so + const bool keysApply = !AnticheatPlugInterface::DoesACPluginProvideSecureGameTransport(); + std::lock_guard lock(m_mapConnectionsMutex); + for (int64_t userID : humans) + { + if (userID == auth->GetUserID()) + { + continue; + } + const PlayerConnection* connection = GetConnectionForUser(userID); + if (connection == nullptr || connection->GetState() != EConnectionState::CONNECTED_DIRECT) + { + blockers.push_back({ userID, true }); + } + else if (keysApply && (!m_matchKeys.HasPeerKey(userID) || !m_matchKeys.PeerHoldsAll(userID, humans))) + { + blockers.push_back({ userID, false }); + } + } +} + +void NetworkMesh::OnKeyAnnouncement(int64_t connectionUserID, const uint8_t* data, size_t len) +{ + if (data == nullptr || len == 0 || !m_matchKeys.HasLocalKey()) + { + return; + } + + // in the lobby only its members count, a sender whose member entry has not arrived yet announces again + if (!m_matchKeys.IsActive()) + { + NGMP_OnlineServices_LobbyInterface* lobby = NGMP_OnlineServicesManager::GetInterface(); + bool member = false; + if (lobby != nullptr) + { + for (const LobbyMemberEntry& entry : lobby->GetCurrentLobby().members) + { + member |= entry.IsHuman() && entry.user_id == connectionUserID; + } + } + if (!member) + { + return; + } + } + + std::lock_guard lock(m_mapConnectionsMutex); + const char* szReason = nullptr; + bool changed = false; + const bool accepted = data[0] == MatchKeys::KIND_STATUS + ? m_matchKeys.OnStatus(connectionUserID, data, len, szReason, changed) + : m_matchKeys.OnAnnouncement(connectionUserID, data, len, szReason, changed); + if (!accepted) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[KEYS] Rejected a key message from user %lld: %s", static_cast(connectionUserID), szReason != nullptr ? szReason : "unknown"); + return; + } + + if (changed && data[0] == MatchKeys::KIND_ANNOUNCEMENT) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[KEYS] have keys for %zu peers, got one from user %lld", m_matchKeys.GetPeerKeyCount(), static_cast(connectionUserID)); + } +} + void NetworkMesh::AwaitTurnCredentials() { std::lock_guard lock(m_mapConnectionsMutex); @@ -1130,6 +1441,10 @@ void NetworkMesh::Disconnect() m_bDisconnected = true; + EndMatchKeys(); + m_matchKeys.End(); + m_keySync.clear(); + for (auto& connectionData : m_mapConnections) { connectionData.second.Close(); @@ -1188,6 +1503,8 @@ void NetworkMesh::Tick() conn.UpdateLatencyHistogram(); } + TickLobbyKeys(nowMs); + // the game transport isn't created until the game begins, but we want to transfer AC packets in the lobby first, so consider this a liteupdate if (TheNGMPGame != nullptr && !TheNGMPGame->isGameInProgress()) { @@ -1264,6 +1581,12 @@ void PlayerConnection::LiteUpdateForAC() msg->Release(); continue; } + else if (netChannel == ENetworkChannel::NETWORK_CHANNEL_KEYS) + { + HandleKeyMessage(vecData.data() + sizeof(ENetworkChannel), numBytes - sizeof(ENetworkChannel)); + msg->Release(); + continue; + } } else if (numBytes != -1 && numBytes < sizeof(ENetworkChannel)) { @@ -1365,6 +1688,15 @@ void PlayerConnection::TickProbe(int64_t nowMs) slot.sendTimeMs = nowMs; } +void PlayerConnection::HandleKeyMessage(const uint8_t* data, size_t len) +{ + NetworkMesh* pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + if (pMesh != nullptr) + { + pMesh->OnKeyAnnouncement(m_userID, data, len); + } +} + int64_t PlayerConnection::GetArrivalMs(const SteamNetworkingMessage_t* msg) { const int64_t nowMs = NowSteadyMs(); diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp index 78fac2e4447..404094f7c7d 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp @@ -399,6 +399,12 @@ Bool NextGenTransport::doRecv(void) msg->Release(); continue; } + else if (netChannel == ENetworkChannel::NETWORK_CHANNEL_KEYS) + { + kvPair.second.HandleKeyMessage(reinterpret_cast(vecPacketDataWithoutHeader.data()), vecPacketDataWithoutHeader.size()); + msg->Release(); + continue; + } else if (netChannel == ENetworkChannel::NETWORK_CHANNEL_PING) { kvPair.second.HandleProbe(reinterpret_cast(vecPacketDataWithoutHeader.data()), vecPacketDataWithoutHeader.size(), PlayerConnection::GetArrivalMs(msg)); @@ -548,6 +554,10 @@ Bool NextGenTransport::doRecv(void) // Clear slot std::memset(&m_inBuffer[i], 0, sizeof(m_inBuffer[i])); + // the connection the packet arrived on says which user sent it + m_inBuffer[i].senderKnown = true; + m_inBuffer[i].senderUserID = static_cast(kvPair.second.m_userID); + // Copy header m_inBuffer[i].header = incomingMessage.header; diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_LobbyInterface.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_LobbyInterface.cpp index 4b5b9da9cbc..ec1a32ef2f0 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_LobbyInterface.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_LobbyInterface.cpp @@ -828,6 +828,10 @@ void NGMP_OnlineServices_LobbyInterface::Tick() if (!TheNGMPGame->HasCommittedOutcome()) { TheNGMPGame->SetHasCommittedOutcome(); + if (m_pLobbyMesh != nullptr) + { + m_pLobbyMesh->EndMatchKeys(); + } pStatsInterface->CommitMyOutcome(localPlayer->getScoreKeeper(), TheVictoryConditions->isLocalAlliedVictory()); } } diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp index 07e13867223..3374828c534 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp @@ -1414,7 +1414,8 @@ void WebSocket::Tick() { NetworkLog(ELogVerbosity::LOG_RELEASE, "[SIGNAL] Signal User: %lld!", signalData.target_user_id); NetworkLog(ELogVerbosity::LOG_RELEASE, "[SIGNAL] Signal Payload Size: %d!", (int)signalData.payload.size()); - PushPendingSignal(std::move(signalData.payload)); + // target_user_id is the sender here, the service fills it in from the authenticated session + PushPendingSignal(signalData.target_user_id, std::move(signalData.payload)); } } break; From ee259f3538eee8906231296ac0eb68cbaf1e5966 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Thu, 22 Oct 2026 07:03:27 +0200 Subject: [PATCH 08/12] feat(network): Send commands directly across the peer mesh --- .../Include/GameNetwork/ConnectionManager.h | 8 +- .../Include/GameNetwork/FrameMetrics.h | 1 + .../Include/GameNetwork/NetworkSettings.h | 1 + .../Source/GameNetwork/ConnectionManager.cpp | 103 +++++++++++++++++- .../Source/GameNetwork/FrameMetrics.cpp | 4 + .../GeneralsOnline/OnlineServices_Init.h | 1 + 6 files changed, 111 insertions(+), 7 deletions(-) diff --git a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h index 7ec23934856..05130873681 100644 --- a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h +++ b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h @@ -255,8 +255,14 @@ class ConnectionManager Bool isDuplicateCommand(NetCommandMsg *msg); Int slotForUserID(Int64 userID) const; + Bool isPeerActive(Int slot) const; + void onDirectFrameAck(UnsignedInt frame, Int peerSlot); + std::map m_frameAckedMasks; ///< Frame -> the peers that acked its frame info so far. + Bool isDirectSendActive(); + Bool m_directSendActive; + time_t m_directSendCheckedAt; UnsignedInt m_formerRouterMask; ///< Slots that were the packet router in this match, their last packets can still be on the way. - Bool isCommandFromSender(NetCommandMsg *msg, Int senderSlot); ///< A command must come from the player it names, unless the packet router relays it + Bool isCommandFromSender(NetCommandMsg *msg, Int senderSlot); Bool noteSecurityDrop(); Bool m_loggedSenderUnavailable; Int m_securityDrops; diff --git a/Core/GameEngine/Include/GameNetwork/FrameMetrics.h b/Core/GameEngine/Include/GameNetwork/FrameMetrics.h index fdf4edca419..f717b2ba03d 100644 --- a/Core/GameEngine/Include/GameNetwork/FrameMetrics.h +++ b/Core/GameEngine/Include/GameNetwork/FrameMetrics.h @@ -50,6 +50,7 @@ class FrameMetrics { Real getAverageLatency(); Int getAverageFPS(); Int getMinimumCushion(); + Bool isPending(UnsignedInt frame); void SeedLatencyData(int latency); diff --git a/Core/GameEngine/Include/GameNetwork/NetworkSettings.h b/Core/GameEngine/Include/GameNetwork/NetworkSettings.h index 28f10cecbdd..eb4cf850d8e 100644 --- a/Core/GameEngine/Include/GameNetwork/NetworkSettings.h +++ b/Core/GameEngine/Include/GameNetwork/NetworkSettings.h @@ -10,6 +10,7 @@ struct NetworkSettings struct Send { std::string mode = "datagram"; // "reliable" = GNS retransmits, "datagram" = unreliable datagrams + bool direct = true; // every player sends commands straight to each peer while the mesh is up int frames_per_packet = 2; // send interval in frames int frames_per_packet_lossy = 1; // send interval in frames while the redundancy depth is above 0, 0 = keep the configured one bool flush_every_frame = true; // flush GNS after each frame diff --git a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp index 5237e9a9c32..5191bbce8b3 100644 --- a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp +++ b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp @@ -358,7 +358,10 @@ void ConnectionManager::init() m_lastKnownRunAhead = 0; m_loggedSenderUnavailable = FALSE; m_securityDrops = 0; + m_directSendActive = FALSE; + m_frameAckedMasks.clear(); m_formerRouterMask = 0; + m_directSendCheckedAt = 0; clearSignatureState(TRUE); m_frameGroupingMs = 0; #endif @@ -554,6 +557,39 @@ Bool ConnectionManager::isDuplicateCommand(NetCommandMsg *msg) { return FALSE; } +enum { ACK_PRUNE_FRAMES = 256, DIRECT_RECHECK_MS = 250 }; + +Bool ConnectionManager::isPeerActive(Int slot) const { + return m_connections[slot] != nullptr && !m_connections[slot]->isQuitting(); +} + +// Frame infos sent directly are acked by every peer, the latency of a frame is the slowest peer's round trip. +void ConnectionManager::onDirectFrameAck(UnsignedInt frame, Int peerSlot) { + if (peerSlot < 0 || peerSlot >= MAX_SLOTS || !m_frameMetrics.isPending(frame)) { + return; + } + + UnsignedInt &acked = m_frameAckedMasks[frame]; + acked |= (1u << peerSlot); + + UnsignedInt expected = 0; + for (Int slot = 0; slot < MAX_SLOTS; ++slot) { + if (slot != m_localSlot && isPeerActive(slot)) { + expected |= (1u << slot); + } + } + + if ((acked & expected) == expected) { + m_frameMetrics.processLatencyResponse(frame); + m_frameAckedMasks.erase(frame); + } + + // frames that never completed, a peer's ack was lost for good + while (!m_frameAckedMasks.empty() && m_frameAckedMasks.begin()->first + ACK_PRUNE_FRAMES < frame) { + m_frameAckedMasks.erase(m_frameAckedMasks.begin()); + } +} + Int ConnectionManager::slotForUserID(Int64 userID) const { return (TheNGMPGame != nullptr) ? TheNGMPGame->getSlotOfUser(userID) : -1; } @@ -589,6 +625,38 @@ static Int relayClass(NetCommandType type) { } } +// send.direct is on and every peer has a direct connection; commands then bypass the packet router. +Bool ConnectionManager::isDirectSendActive() { + if (TheNGMPGame == nullptr || !m_networkSettings.send.direct) { + return FALSE; + } + + const time_t now = timeGetTime(); + if (m_directSendCheckedAt != 0 && (now - m_directSendCheckedAt) < DIRECT_RECHECK_MS) { + return m_directSendActive; + } + m_directSendCheckedAt = now; + + Bool meshUp = TRUE; + NetworkMesh *mesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + for (Int slot = 0; slot < MAX_SLOTS && meshUp; ++slot) { + if (slot == m_localSlot || !isPeerActive(slot)) { + continue; + } + const NGMPGameSlot *gameSlot = (const NGMPGameSlot *)TheNGMPGame->getConstSlot(slot); + PlayerConnection *connection = (mesh != nullptr && gameSlot != nullptr) ? mesh->GetConnectionForUser(gameSlot->m_userID) : nullptr; + if (connection == nullptr || connection->GetState() != EConnectionState::CONNECTED_DIRECT) { + meshUp = FALSE; + } + } + + if (meshUp != m_directSendActive) { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[DIRECT] commands are now sent %s", meshUp ? "directly to every peer" : "through the packet router, a peer connection is not up"); + } + m_directSendActive = meshUp; + return m_directSendActive; +} + // Checks that a command names its sender, or that the packet router or a former one passes it on. Bool ConnectionManager::isCommandFromSender(NetCommandMsg *msg, Int senderSlot) { const Int claimedSlot = (Int)msg->getPlayerID(); @@ -827,7 +895,8 @@ void ConnectionManager::signLocalFrame(NetFrameCommandMsg *frameInfo) { void ConnectionManager::signLocalChat(NetChatCommandMsg *chat) { MatchKeys *keys = activeMatchKeys(); - if (keys == nullptr || m_packetRouterSlot >= MAX_SLOTS || m_packetRouterSlot == m_localSlot || m_connections[m_packetRouterSlot] == nullptr) { + if (keys == nullptr || m_packetRouterSlot >= MAX_SLOTS || m_packetRouterSlot == m_localSlot || m_connections[m_packetRouterSlot] == nullptr + || isDirectSendActive()) { return; } @@ -1736,8 +1805,16 @@ void ConnectionManager::processAckStage1(NetCommandMsg *msg) { } if (ref != nullptr) { - if (ref->getCommand()->getNetCommandType() == NETCOMMANDTYPE_FRAMEINFO) { - m_frameMetrics.processLatencyResponse(((NetFrameCommandMsg *)(ref->getCommand()))->getExecutionFrame()); + if (ref->getCommand()->getNetCommandType() == NETCOMMANDTYPE_FRAMEINFO || ref->getCommand()->getNetCommandType() == NETCOMMANDTYPE_SIGNEDFRAMEINFO) { + const UnsignedInt ackedFrame = ref->getCommand()->getExecutionFrame(); +#if defined(GENERALS_ONLINE) + if (isDirectSendActive()) { + onDirectFrameAck(ackedFrame, (Int)playerID); + } else +#endif + { + m_frameMetrics.processLatencyResponse(ackedFrame); + } } deleteInstance(ref); @@ -1950,7 +2027,16 @@ void ConnectionManager::ackCommand(NetCommandRef *ref, UnsignedInt localSlot) { ackmsg->setPlayerID(localSlot); // Tell the player who this ack is coming from. - if (CommandRequiresDirectSend(msg) && CommandRequiresAck(msg)) { + // A command that came straight from its author and needs no relaying is acked straight back to it, one the router + // passed on is acked to the router. The arrival decides, not our own direct send mode: peers can be in different modes. + Bool ackDirectly = CommandRequiresDirectSend(msg) && CommandRequiresAck(msg); +#if defined(GENERALS_ONLINE) && !RETAIL_COMPATIBLE_NETWORKING + if (!ackDirectly && sendRelay == 0 && msg->getPlayerID() < MAX_SLOTS && m_connections[msg->getPlayerID()] != nullptr) { + // an unknown sender (anticheat transport) falls back to our own mode + ackDirectly = (m_currentSenderSlot >= 0) ? (m_currentSenderSlot == (Int)msg->getPlayerID()) : isDirectSendActive(); + } +#endif + if (ackDirectly) { // Send this ack directly back to the sending player, don't go through the packet router. if (msg->getPlayerID() < MAX_SLOTS) { if (m_connections[msg->getPlayerID()] != nullptr) { @@ -2592,8 +2678,13 @@ void ConnectionManager::sendLocalCommand(NetCommandMsg *msg, UnsignedByte relay } // Send the packet to everyone else - if (m_localSlot == m_packetRouterSlot) { - // I am the packet router, I need to send this packet to everyone individually. +#if defined(GENERALS_ONLINE) + const Bool sendToAll = isDirectSendActive() || (m_localSlot == m_packetRouterSlot); +#else + const Bool sendToAll = (m_localSlot == m_packetRouterSlot); +#endif + if (sendToAll) { + // I am the packet router or sending directly, I need to send this packet to everyone individually. for (Int i = 0; i < MAX_SLOTS; ++i) { // Send it to all open connections. if (((m_connections[i] != nullptr) && (m_connections[i]->isQuitting() == FALSE)) && (relay & (1 << i))) { diff --git a/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp b/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp index abc805ddb71..16e45c9e367 100644 --- a/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp +++ b/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp @@ -285,6 +285,10 @@ Real FrameMetrics::getAverageLatency() { return m_averageLatency; } +Bool FrameMetrics::isPending(UnsignedInt frame) { + return m_mapPendingLatenciesLookup.find(frame) != m_mapPendingLatenciesLookup.end(); +} + Int FrameMetrics::getMinimumCushion() { return m_minimumCushion; } diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index 3eb1c9dbb42..1277851dc6d 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -433,6 +433,7 @@ inline void ReadServiceConfig(const nlohmann::json& j, ServiceConfig& c) { const nlohmann::json& s = j["send"]; ReadServiceConfigValue(s, "mode", c.send.mode); + ReadServiceConfigValue(s, "direct", c.send.direct); ReadServiceConfigValue(s, "frames_per_packet", c.send.frames_per_packet); ReadServiceConfigValue(s, "frames_per_packet_lossy", c.send.frames_per_packet_lossy); ReadServiceConfigValue(s, "flush_every_frame", c.send.flush_every_frame); From fbb383a8e7416d734374700e4b1c0f350cb6ad9d Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Thu, 22 Oct 2026 07:03:28 +0200 Subject: [PATCH 09/12] fix(network): Synchronize disconnect removal and quit handling --- .../Include/GameNetwork/ConnectionManager.h | 13 +- .../Include/GameNetwork/DisconnectManager.h | 33 +- .../Include/GameNetwork/NetworkInterface.h | 2 +- .../Include/GameNetwork/NetworkSettings.h | 6 + .../Source/GameNetwork/Connection.cpp | 3 +- .../Source/GameNetwork/ConnectionManager.cpp | 151 ++++++- .../Source/GameNetwork/DisconnectManager.cpp | 371 +++++++++++++++++- .../Source/GameNetwork/NetPacket.cpp | 3 + .../GameEngine/Source/GameNetwork/Network.cpp | 7 + .../GameNetwork/GeneralsOnline/NetworkMesh.h | 2 + .../GeneralsOnline/NextGenTransport.h | 3 + .../GeneralsOnline/OnlineServices_Init.h | 11 + .../GeneralsOnline/NetworkMesh.cpp | 33 +- .../GeneralsOnline/NextGenTransport.cpp | 42 +- 14 files changed, 643 insertions(+), 37 deletions(-) diff --git a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h index 05130873681..dd2268297e1 100644 --- a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h +++ b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h @@ -112,6 +112,7 @@ class ConnectionManager UnsignedInt getPacketRouterSlot(); ///< Returns the current packet router's slot. PlayerLeaveCode disconnectPlayer(Int slot); ///< Disconnect this player immediately. This should only be called by the disconnect manager. #if defined(GENERALS_ONLINE) + void playerClosedGracefully(Int64 userID); PlayerLeaveCode disconnectPlayer(int64_t userID); ///< Disconnect this player immediately. This should only be called by the disconnect manager. #endif void disconnectLocalPlayer(); ///< Does whatever is necessary to get TheNetwork to realize that it should be leaving the game now. @@ -151,7 +152,7 @@ class ConnectionManager void notifyOthersOfCurrentFrame(Int frame); void sendFrameDataToPlayer(UnsignedInt playerID, UnsignedInt startingFrame); - void sendSingleFrameToPlayer(UnsignedInt playerID, UnsignedInt frame); + void sendSingleFrameToPlayer(UnsignedInt playerID, UnsignedInt frame, Bool ownOnly = FALSE); void notifyOthersOfNewFrame(UnsignedInt frame); UnsignedInt getNextPacketRouterSlot(UnsignedInt playerID); ///< returns the packet router player that comes after the given player. @@ -204,6 +205,11 @@ class ConnectionManager Real getMaximumLatency(); ///< Returns the average of the two highest average latencies between players. void requestFrameDataResend(Int playerID, UnsignedInt frame); ///< request of this player that he send the specified frame's data. + UnsignedShort generateImpersonatedCommandID(Int playerID, UnsignedInt frame); +#if !RETAIL_COMPATIBLE_NETWORKING + void askForOutPlayerFrame(Int slot, UnsignedInt frame); + Int getSlotOfAddress(UnsignedInt addr, UnsignedShort port); +#endif // The connections are set up like the slot list. The connection corresponding to the local // player's position in the slot list will be null. Connections corresponding to slots that @@ -224,8 +230,13 @@ class ConnectionManager NetCommandList *m_pendingCommands; NetCommandList *m_relayedCommands; + UnsignedShort m_lastSeenCommandID[MAX_SLOTS]; + Bool m_haveLastSeenCommandID[MAX_SLOTS]; #if !RETAIL_COMPATIBLE_NETWORKING Int m_currentSenderSlot; ///< The slot of the peer the packet being read came from, -1 when not known. + Int m_outFrameAskedSlot; + UnsignedInt m_outFrameAskedFrame; + UnsignedInt m_outFrameAskedTime; #endif FrameMetrics m_frameMetrics; diff --git a/Core/GameEngine/Include/GameNetwork/DisconnectManager.h b/Core/GameEngine/Include/GameNetwork/DisconnectManager.h index ccbb183881f..fc6b6a5fa38 100644 --- a/Core/GameEngine/Include/GameNetwork/DisconnectManager.h +++ b/Core/GameEngine/Include/GameNetwork/DisconnectManager.h @@ -58,6 +58,13 @@ class DisconnectManager void voteForPlayerDisconnect(Int slot, ConnectionManager *conMgr); +#if !RETAIL_COMPATIBLE_NETWORKING + void playerClosedGracefully(Int slot, ConnectionManager *conMgr); ///< The next packet router removes the player without waiting out its timeout. + Bool isPlayerOutOfGame(Int slot, ConnectionManager *conMgr); +#endif + + Bool isLocalPlayerNextPacketRouter(ConnectionManager *conMgr); ///< returns true if the local player is next in line to be the packet router with all the players that have timed out being taken out of the picture. + // For disconnect blame assignment UnsignedInt getPingFrame(); Int getPingsSent(); @@ -78,12 +85,23 @@ class DisconnectManager void updateWaitForPacketRouter(ConnectionManager *conMgr); void recalculatePacketRouterIndex(ConnectionManager *conMgr); Bool allOnSameFrame(ConnectionManager *conMgr); ///< returns true if all players are stuck on the same frame. - Bool isLocalPlayerNextPacketRouter(ConnectionManager *conMgr); ///< returns true if the local player is next in line to be the packet router with all the players that have timed out being taken out of the picture. +#if !RETAIL_COMPATIBLE_NETWORKING + Int getActingPacketRouterSlot(ConnectionManager *conMgr); +#endif Bool hasPlayerTimedOut(Int slot); ///< returns true if this player has timed out. void sendPlayerDestruct(Int slot, ConnectionManager *conMgr); ///< send a destruct player network message. +#if RETAIL_COMPATIBLE_NETWORKING + void sendPendingRelays(ConnectionManager *conMgr); +#endif + void removePlayer(Int slot, ConnectionManager *conMgr); + Bool isPlayerSilent(Int slot); ///< The slot is a translated one. + Bool isPlayerGone(Int slot, ConnectionManager *conMgr); ///< A silent player that timed out or was voted out by all others. The slot is a translated one. Bool isPlayerVotedOut(Int slot, ConnectionManager *conMgr); ///< returns true if this player has been voted out. Bool isPlayerInGame(Int slot, ConnectionManager *conMgr); ///< returns true if the player has neither timed out or been voted out. - UnsignedInt getMaxDisconnectFrame(); ///< returns the highest frame that people have got to. + UnsignedInt getMaxDisconnectFrame(ConnectionManager *conMgr); ///< The highest frame that the players that have not left have got to. +#if !RETAIL_COMPATIBLE_NETWORKING + Bool isGonePlayerConnected(ConnectionManager *conMgr); +#endif Int countVotesForPlayer(Int slot); ///< return the number of disconnect votes a player has. void resetPlayersVotes(Int playerID, UnsignedInt frame, ConnectionManager *conMgr); ///< reset the votes for this player. @@ -109,6 +127,10 @@ class DisconnectManager time_t m_lastKeepAliveSendTime; time_t m_playerTimeouts[MAX_SLOTS - 1]; +#if !RETAIL_COMPATIBLE_NETWORKING + Bool m_closedGracefully[MAX_SLOTS - 1]; ///< By translated slot. + Bool m_screenKept; +#endif time_t m_packetRouterTimeout; DisconnectVoteStruct m_playerVotes[MAX_SLOTS][MAX_SLOTS]; @@ -117,6 +139,13 @@ class DisconnectManager UnsignedInt m_disconnectFrames[MAX_SLOTS]; Bool m_disconnectFramesReceived[MAX_SLOTS]; Bool m_haveNotifiedOtherPlayersOfCurrentFrame; +#if RETAIL_COMPATIBLE_NETWORKING + UnsignedInt m_relayFrame[MAX_SLOTS]; + UnsignedByte m_relayTo[MAX_SLOTS]; +#else + Bool m_removedInRound; + UnsignedInt m_catchUpFrame; +#endif time_t m_timeOfDisconnectScreenOn; Int m_pingsSent; diff --git a/Core/GameEngine/Include/GameNetwork/NetworkInterface.h b/Core/GameEngine/Include/GameNetwork/NetworkInterface.h index a72a3a2c460..a63e2f5db19 100644 --- a/Core/GameEngine/Include/GameNetwork/NetworkInterface.h +++ b/Core/GameEngine/Include/GameNetwork/NetworkInterface.h @@ -61,7 +61,7 @@ class NetworkInterface : public SubsystemInterface virtual void setLocalAddress(UnsignedInt ip, UnsignedInt port) = 0; ///< Tell the network what local ip and port to bind to. virtual Bool isFrameDataReady() = 0; ///< Are the commands for the next frame available? #if defined(GENERALS_ONLINE) - virtual Bool isGameExitPending() const = 0; + virtual Bool isGameExitPending() const { return FALSE; } #endif virtual Bool isStalling() = 0; virtual void parseUserList(const GameInfo* game) = 0; ///< Parse a userlist, creating connections diff --git a/Core/GameEngine/Include/GameNetwork/NetworkSettings.h b/Core/GameEngine/Include/GameNetwork/NetworkSettings.h index eb4cf850d8e..32342b7b7ab 100644 --- a/Core/GameEngine/Include/GameNetwork/NetworkSettings.h +++ b/Core/GameEngine/Include/GameNetwork/NetworkSettings.h @@ -34,6 +34,12 @@ struct NetworkSettings } formula; } input_delay; + struct Disconnect + { + int freeze_before_screen_ms = 5000; // stuck on a frame this long, then the disconnect screen comes up + int remove_after_ms = 60000; // a player silent this long on the disconnect screen is removed + } disconnect; + static NetworkSettings LocalDefaults() { return NetworkSettings(); diff --git a/Core/GameEngine/Source/GameNetwork/Connection.cpp b/Core/GameEngine/Source/GameNetwork/Connection.cpp index ffab6ac480f..adf1c22d6bd 100644 --- a/Core/GameEngine/Source/GameNetwork/Connection.cpp +++ b/Core/GameEngine/Source/GameNetwork/Connection.cpp @@ -684,7 +684,8 @@ NetCommandRef * Connection::processAck(UnsignedShort commandID, UnsignedByte ori // Need to check for both the command ID and the player ID. temp = temp->getNext(); } - if (temp == nullptr) { + // an ack for a command not sent yet belongs to another command with the same id + if (temp == nullptr || temp->getTimeLastSent() == -1) { return nullptr; } diff --git a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp index 5191bbce8b3..98dc13e4c4b 100644 --- a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp +++ b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp @@ -265,6 +265,8 @@ ConnectionManager::ConnectionManager() { for (Int i = 0; i < MAX_SLOTS; ++i) { m_frameData[i] = nullptr; + m_lastSeenCommandID[i] = 0; + m_haveLastSeenCommandID[i] = FALSE; } m_transport = nullptr; m_disconnectManager = nullptr; @@ -292,6 +294,7 @@ void ConnectionManager::init() const ServiceConfig &overrides = services->GetServiceConfig(); m_networkSettings.send = overrides.send; m_networkSettings.input_delay = overrides.input_delay; + m_networkSettings.disconnect = overrides.disconnect; } const NetworkSettings::InputDelay &runAhead = m_networkSettings.input_delay; TheWritableGlobalData->m_networkFPSHistoryLength = (std::max)(1, runAhead.fps_history_samples); @@ -300,6 +303,8 @@ void ConnectionManager::init() TheWritableGlobalData->m_networkRunAheadMetricsTime = runAhead.decision_interval_ms; MIN_RUNAHEAD = runAhead.min_frames; FRAME_GROUPING_CAP = (int)(m_networkSettings.send.frames_per_packet * (float)(1000.f / GENERALS_ONLINE_HIGH_FPS_LIMIT)); + TheWritableGlobalData->m_networkDisconnectTime = m_networkSettings.disconnect.freeze_before_screen_ms; + TheWritableGlobalData->m_networkPlayerTimeoutTime = m_networkSettings.disconnect.remove_after_ms; #endif // if (m_transport == nullptr) { @@ -331,6 +336,8 @@ void ConnectionManager::init() m_packetRouterSlot = 0; /// @todo The LAN/WOL interface should be telling us who the packet router is based on machine specs passed around through game options. for (i = 0; i < MAX_SLOTS; ++i) { m_packetRouterFallback[i] = -1; + m_lastSeenCommandID[i] = 0; + m_haveLastSeenCommandID[i] = FALSE; } for (i = 0; i < MAX_SLOTS; ++i) { @@ -350,6 +357,9 @@ void ConnectionManager::init() m_smallestPacketArrivalCushion = -1; #if !RETAIL_COMPATIBLE_NETWORKING m_currentSenderSlot = -1; + m_outFrameAskedSlot = -1; + m_outFrameAskedFrame = 0; + m_outFrameAskedTime = 0; #endif #if defined(GENERALS_ONLINE) m_stallActive = FALSE; @@ -643,6 +653,12 @@ Bool ConnectionManager::isDirectSendActive() { if (slot == m_localSlot || !isPeerActive(slot)) { continue; } +#if !RETAIL_COMPATIBLE_NETWORKING + // a player that is out needs no commands + if (m_disconnectManager != nullptr && m_disconnectManager->isPlayerOutOfGame(slot, this)) { + continue; + } +#endif const NGMPGameSlot *gameSlot = (const NGMPGameSlot *)TheNGMPGame->getConstSlot(slot); PlayerConnection *connection = (mesh != nullptr && gameSlot != nullptr) ? mesh->GetConnectionForUser(gameSlot->m_userID) : nullptr; if (connection == nullptr || connection->GetState() != EConnectionState::CONNECTED_DIRECT) { @@ -657,7 +673,7 @@ Bool ConnectionManager::isDirectSendActive() { return m_directSendActive; } -// Checks that a command names its sender, or that the packet router or a former one passes it on. +// Checks that a command names its sender, or that a router, a former router or a peer holding a removed player's frames passes it on. Bool ConnectionManager::isCommandFromSender(NetCommandMsg *msg, Int senderSlot) { const Int claimedSlot = (Int)msg->getPlayerID(); if (claimedSlot == senderSlot && claimedSlot >= 0 && claimedSlot < MAX_SLOTS) { @@ -671,10 +687,22 @@ Bool ConnectionManager::isCommandFromSender(NetCommandMsg *msg, Int senderSlot) } else if (nameOk && cls == 2) { maySendForOthers = (senderSlot == m_packetRouterSlot); } +#if !RETAIL_COMPATIBLE_NETWORKING + // a player that is out sends nothing more, so its last signed frames may come from any peer that has them + if (nameOk && cls == 1 && !maySendForOthers && m_disconnectManager != nullptr && m_disconnectManager->isPlayerOutOfGame(claimedSlot, this)) { + maySendForOthers = TRUE; + } +#endif // forwarded frames are checked in isPassedOnFrameAuthentic if (maySendForOthers) { return TRUE; } + // late frames of a removed player: acked, then ignored + if (nameOk && cls == 1 && m_connections[claimedSlot] == nullptr + && m_signerUserID[claimedSlot] >= 0 && IsCommandSynchronized(msg->getNetCommandType())) { + return TRUE; + } + if (noteSecurityDrop()) { NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] dropped %s claiming slot %d from slot %d (%d dropped so far)", GetNetCommandTypeAsString(msg->getNetCommandType()), claimedSlot, (Int)senderSlot, m_securityDrops); @@ -1202,6 +1230,8 @@ void ConnectionManager::doRelay() { #if !RETAIL_COMPATIBLE_NETWORKING #if defined(GENERALS_ONLINE) m_currentSenderSlot = (senderSlot >= 0) ? senderSlot : -1; +#else + m_currentSenderSlot = getSlotOfAddress(m_transport->m_inBuffer[i].addr, m_transport->m_inBuffer[i].port); #endif #endif @@ -1295,6 +1325,16 @@ Bool ConnectionManager::processNetCommand(NetCommandRef *ref) { NetCommandMsg *msg = ref->getCommand(); NetCommandType cmdType = msg->getNetCommandType(); + // Track the command ids of each player, the ids far away from them are free for passed on frame infos. + if (DoesCommandRequireACommandID(cmdType) && msg->getPlayerID() < MAX_SLOTS && msg->getPlayerID() != m_localSlot) { + const UnsignedInt from = msg->getPlayerID(); + const UnsignedShort distance = (UnsignedShort)(msg->getID() - m_lastSeenCommandID[from]); + if (m_haveLastSeenCommandID[from] == FALSE || distance < 0x4000 || distance > 0xC000) { + m_lastSeenCommandID[from] = msg->getID(); + m_haveLastSeenCommandID[from] = TRUE; + } + } + // Handle ACK commands first (before connection validation) if ((cmdType == NETCOMMANDTYPE_ACKSTAGE1) || (cmdType == NETCOMMANDTYPE_ACKSTAGE2) || @@ -2027,6 +2067,15 @@ void ConnectionManager::ackCommand(NetCommandRef *ref, UnsignedInt localSlot) { ackmsg->setPlayerID(localSlot); // Tell the player who this ack is coming from. +#if !RETAIL_COMPATIBLE_NETWORKING + // a peer other than the router that passed on another player's command waits for the ack itself + if ((m_currentSenderSlot >= 0) && (m_currentSenderSlot != (Int)msg->getPlayerID()) && (m_currentSenderSlot != (Int)m_packetRouterSlot) && (m_connections[m_currentSenderSlot] != nullptr)) { + m_connections[m_currentSenderSlot]->sendNetCommandMsg(ackmsg, 1 << m_currentSenderSlot); + ackmsg->detach(); + return; + } +#endif + // A command that came straight from its author and needs no relaying is acked straight back to it, one the router // passed on is acked to the router. The arrival decides, not our own direct send mode: peers can be in different modes. Bool ackDirectly = CommandRequiresDirectSend(msg) && CommandRequiresAck(msg); @@ -2767,6 +2816,11 @@ Bool ConnectionManager::allCommandsReady(UnsignedInt frame, Bool justTesting /* frameRetVal = m_frameData[i]->allCommandsReady(frame, (frame != commandsReadyDebugSpewage) && (justTesting == FALSE)); if (frameRetVal == FRAMEDATA_NOTREADY) { retval = FALSE; +#if !RETAIL_COMPATIBLE_NETWORKING + if ((justTesting == FALSE) && (i != (Int)m_localSlot)) { + askForOutPlayerFrame(i, frame); + } +#endif } else if (frameRetVal == FRAMEDATA_RESEND) { requestFrameDataResend(i, frame); retval = FALSE; @@ -2991,8 +3045,23 @@ PlayerLeaveCode ConnectionManager::disconnectPlayer(Int slot) { } #if defined(GENERALS_ONLINE) +void ConnectionManager::playerClosedGracefully(Int64 userID) +{ + Int slot = slotForUserID(userID); + if (slot >= 0 && slot < MAX_SLOTS && m_disconnectManager != nullptr) + { + m_disconnectManager->playerClosedGracefully(slot, this); + } +} + PlayerLeaveCode ConnectionManager::disconnectPlayer(int64_t userID) { + // a player whose connection ended before the load finished must not hold the others at the load screen, the disconnect handling removes it once the match runs + const Int slot = slotForUserID(userID); + if (slot >= 0 && slot != (Int)m_localSlot && TheGameLogic != nullptr && TheGameLogic->isInInternetGame() && !TheGameLogic->isProgressComplete()) { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[LOAD] slot %d is gone before the load finished, not waiting for it", (Int)slot); + TheGameLogic->processProgressComplete(slot); + } return PLAYERLEAVECODE_UNKNOWN; // if (TheNGMPGame != nullptr) // { @@ -3022,9 +3091,15 @@ void ConnectionManager::quitGame() { if (DoesCommandRequireACommandID(disconnectMsg->getNetCommandType())) { disconnectMsg->setID(GenerateNextCommandID()); } +#if defined(GENERALS_ONLINE) + signLeaveNotice(disconnectMsg); // lets the peers pass the notice on +#endif //DEBUG_LOG(("ConnectionManager::disconnectLocalPlayer - about to send disconnect command")); sendLocalCommandDirect(disconnectMsg, 0xff ^ (1 << m_localSlot)); + // the connections are deleted right after, so send now + setFrameGrouping(0); + //DEBUG_LOG(("ConnectionManager::disconnectLocalPlayer - about to flush connections")); flushConnections(); // need to do this so our packet actually gets sent before the connections are deleted. //DEBUG_LOG(("ConnectionManager::disconnectLocalPlayer - done flushing connections")); @@ -3370,6 +3445,8 @@ void ConnectionManager::sendDisconnectChat(UnicodeString text) { sendLocalCommandDirect(msg, 0xff ^ (1 << m_localSlot)); processDisconnectChat(msg); + + msg->detach(); } UnsignedShort ConnectionManager::sendFileAnnounce(AsciiString path, UnsignedByte playerMask) @@ -3644,15 +3721,26 @@ void ConnectionManager::notifyOthersOfNewFrame(UnsignedInt frame) { msg->detach(); } +// An id half the counter away from the player's own, so a passed on frame info never collides with its commands. +UnsignedShort ConnectionManager::generateImpersonatedCommandID(Int playerID, UnsignedInt frame) { + return (UnsignedShort)(((m_lastSeenCommandID[playerID] + 0x8000) & 0xF000) | (frame & 0x0FFF)); +} + void ConnectionManager::sendFrameDataToPlayer(UnsignedInt playerID, UnsignedInt startingFrame) { DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("ConnectionManager::sendFrameDataToPlayer - sending frame data to player %d starting with frame %d", playerID, startingFrame)); for (UnsignedInt frame = startingFrame; frame < TheGameLogic->getFrame(); ++frame) { sendSingleFrameToPlayer(playerID, frame); } +#if defined(GENERALS_ONLINE) + // a peer that dropped a passed on frame asks for it while we have not run it yet: our own data for it is final + if (startingFrame >= TheGameLogic->getFrame() && startingFrame <= m_lastFrameInfoSent) { + sendSingleFrameToPlayer(playerID, startingFrame, TRUE); + } +#endif DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("ConnectionManager::sendFrameDataToPlayer - done sending commands to player %d", playerID)); } -void ConnectionManager::sendSingleFrameToPlayer(UnsignedInt playerID, UnsignedInt frame) { +void ConnectionManager::sendSingleFrameToPlayer(UnsignedInt playerID, UnsignedInt frame, Bool ownOnly) { if (frame + FRAMES_TO_KEEP < TheGameLogic->getFrame()) { DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("ConnectionManager::sendSingleFrameToPlayer - player %d requested frame %d when we are on frame %d, this is too far in the past.", playerID, frame, TheGameLogic->getFrame())); return; @@ -3660,8 +3748,31 @@ void ConnectionManager::sendSingleFrameToPlayer(UnsignedInt playerID, UnsignedIn UnsignedByte relay = 1 << playerID; +#if !RETAIL_COMPATIBLE_NETWORKING + const Bool isActingRouter = m_disconnectManager->isLocalPlayerNextPacketRouter(this); +#endif + DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("ConnectionManager::sendFrameDataToPlayer - sending data for frame %d", frame)); for (Int i = 0; i < MAX_SLOTS; ++i) { +#if !RETAIL_COMPATIBLE_NETWORKING + // Own data and that of a player that is out go from everyone, the rest only from the acting packet router. + if ((isActingRouter == FALSE) && (i != (Int)m_localSlot) && (m_disconnectManager->isPlayerOutOfGame(i, this) == FALSE)) { + continue; + } +#endif +#if defined(GENERALS_ONLINE) + // For a frame we have not run yet only our own data is final, and that of a player that is out once we hold all of it. + if (ownOnly && (i != (Int)m_localSlot)) { + Bool finalOfOutPlayer = FALSE; +#if !RETAIL_COMPATIBLE_NETWORKING + finalOfOutPlayer = (m_frameData[i] != nullptr) && m_disconnectManager->isPlayerOutOfGame(i, this) + && (m_frameData[i]->getFrameCommandCount(frame) == m_frameData[i]->getCommandCount(frame)); +#endif + if (!finalOfOutPlayer) { + continue; + } + } +#endif if ((m_frameData[i] != nullptr) && (i != playerID)) { // no need to send his own commands to him. NetCommandList *list = m_frameData[i]->getFrameCommandList(frame); if (list != nullptr) { @@ -3677,7 +3788,7 @@ void ConnectionManager::sendSingleFrameToPlayer(UnsignedInt playerID, UnsignedIn msg->setExecutionFrame(frame); msg->setCommandCount(frameCommandCount); if (DoesCommandRequireACommandID(msg->getNetCommandType())) { - msg->setID(GenerateNextCommandID()); + msg->setID(i == (Int)m_localSlot ? GenerateNextCommandID() : generateImpersonatedCommandID(i, frame)); } msg->setPlayerID(i); DEBUG_LOG_LEVEL(DEBUG_LEVEL_NET, ("ConnectionManager::sendFrameDataToPlayer - sending frame info from player %d to player %d for frame %d with command count %d and ID %d and relay %d", i, playerID, msg->getExecutionFrame(), msg->getCommandCount(), msg->getID(), relay)); @@ -3712,6 +3823,15 @@ void ConnectionManager::requestFrameDataResend(Int playerID, UnsignedInt frame) msg->setID(GenerateNextCommandID()); } +#if !RETAIL_COMPATIBLE_NETWORKING + // A player that is out answers no more, so the others that may hold its frames are asked instead. + if (m_disconnectManager->isPlayerOutOfGame(playerID, this) == TRUE) { + sendLocalCommandDirect(msg, (UnsignedByte)(0xff & ~(1 << m_localSlot) & ~(1 << playerID))); + msg->detach(); + return; + } +#endif + if (isPlayerConnected(playerID) == FALSE) { playerID = 0; while ((playerID < MAX_SLOTS) && (isPlayerConnected(playerID) == FALSE)) { @@ -3726,6 +3846,31 @@ void ConnectionManager::requestFrameDataResend(Int playerID, UnsignedInt frame) msg->detach(); } +#if !RETAIL_COMPATIBLE_NETWORKING +void ConnectionManager::askForOutPlayerFrame(Int slot, UnsignedInt frame) { + if (m_disconnectManager->isPlayerOutOfGame(slot, this) == FALSE) { + return; + } + const UnsignedInt now = timeGetTime(); + if ((slot == m_outFrameAskedSlot) && (frame == m_outFrameAskedFrame) && ((now - m_outFrameAskedTime) < 1000)) { + return; + } + m_outFrameAskedSlot = slot; + m_outFrameAskedFrame = frame; + m_outFrameAskedTime = now; + requestFrameDataResend(slot, frame); +} + +Int ConnectionManager::getSlotOfAddress(UnsignedInt addr, UnsignedShort port) { + for (Int i = 0; i < MAX_SLOTS; ++i) { + if ((m_connections[i] != nullptr) && (m_connections[i]->getUser()->GetIPAddr() == addr) && (m_connections[i]->getUser()->GetPort() == port)) { + return i; + } + } + return -1; +} +#endif + #if defined(GENERALS_ONLINE) void ConnectionManager::noteStalling(Bool stalling, UnsignedInt frame) { time_t now = timeGetTime(); diff --git a/Core/GameEngine/Source/GameNetwork/DisconnectManager.cpp b/Core/GameEngine/Source/GameNetwork/DisconnectManager.cpp index a59a6fcbc7c..eaab073f2d8 100644 --- a/Core/GameEngine/Source/GameNetwork/DisconnectManager.cpp +++ b/Core/GameEngine/Source/GameNetwork/DisconnectManager.cpp @@ -35,6 +35,18 @@ #include "GameNetwork/GameSpy/PingThread.h" #include "GameNetwork/GameSpy/GSConfig.h" +static const UnsignedInt SILENT_PLAYER_TIME = 2000; + +#if !RETAIL_COMPATIBLE_NETWORKING +static const UnsignedInt SCREEN_NOTIFY_TIME = 1000; + +static const UnsignedInt CATCH_UP_WAIT_TIME = 1000; + +// Frame 0 in a disconnect player command marks a leave notice, never a frame to remove the player on. +static const UnsignedInt LEAVE_NOTICE_FRAME = 0; + +static const UnsignedInt KEPT_SCREEN_WAIT_TIME = 200; +#endif DisconnectManager::DisconnectManager() { @@ -48,12 +60,24 @@ DisconnectManager::DisconnectManager() for( i = 0; i < MAX_SLOTS; ++i) { m_packetRouterFallback[i] = 0; +#if RETAIL_COMPATIBLE_NETWORKING + m_relayFrame[i] = 0; + m_relayTo[i] = 0; +#endif } m_packetRouterTimeout = 0; for( i = 0; i < MAX_SLOTS -1; ++i) { m_playerTimeouts[i] = 0; +#if !RETAIL_COMPATIBLE_NETWORKING + m_closedGracefully[i] = FALSE; +#endif } +#if !RETAIL_COMPATIBLE_NETWORKING + m_screenKept = FALSE; + m_removedInRound = FALSE; + m_catchUpFrame = 0; +#endif for( i = 0; i < MAX_SLOTS; ++i) { for (Int j = 0; j < MAX_SLOTS; ++j) { @@ -86,7 +110,19 @@ void DisconnectManager::init() { for (i = 0; i < MAX_SLOTS; ++i) { m_disconnectFrames[i] = 0; m_disconnectFramesReceived[i] = FALSE; +#if RETAIL_COMPATIBLE_NETWORKING + m_relayFrame[i] = 0; + m_relayTo[i] = 0; +#endif + } +#if !RETAIL_COMPATIBLE_NETWORKING + for (i = 0; i < MAX_SLOTS - 1; ++i) { + m_closedGracefully[i] = FALSE; } + m_screenKept = FALSE; + m_removedInRound = FALSE; + m_catchUpFrame = 0; +#endif m_pingFrame = 0; m_pingsSent = 0; @@ -103,7 +139,17 @@ void DisconnectManager::update(ConnectionManager *conMgr) { // the last one we had the commands ready for. if (TheGameLogic->getFrame() == m_lastFrame) { time_t curTime = timeGetTime(); - if ((curTime - m_lastFrameTime) > TheGlobalData->m_networkDisconnectTime) { +#if RETAIL_COMPATIBLE_NETWORKING + const time_t waitTime = TheGlobalData->m_networkDisconnectTime; +#else + const Bool catchUp = (m_catchUpFrame != 0) && (m_catchUpFrame == TheGameLogic->getFrame()); + time_t waitTime = catchUp ? std::min(TheGlobalData->m_networkDisconnectTime, CATCH_UP_WAIT_TIME) : TheGlobalData->m_networkDisconnectTime; + // screen kept for a leaver: no catch-up wait + if (m_screenKept == TRUE) { + waitTime = std::min(waitTime, (time_t)KEPT_SCREEN_WAIT_TIME); + } +#endif + if ((curTime - m_lastFrameTime) > waitTime) { if (m_disconnectState == DISCONNECTSTATETYPE_SCREENOFF) { turnOnScreen(conMgr); } @@ -113,6 +159,17 @@ void DisconnectManager::update(ConnectionManager *conMgr) { nextFrame(TheGameLogic->getFrame(), conMgr); } +#if !RETAIL_COMPATIBLE_NETWORKING + if ((m_screenKept == TRUE) && (m_disconnectState == DISCONNECTSTATETYPE_SCREENOFF) && (isGonePlayerConnected(conMgr) == FALSE)) { + TheDisconnectMenu->hideScreen(); + m_screenKept = FALSE; + } +#endif + +#if RETAIL_COMPATIBLE_NETWORKING + sendPendingRelays(conMgr); +#endif + if (m_disconnectState != DISCONNECTSTATETYPE_SCREENOFF) { updateDisconnectStatus(conMgr); @@ -198,12 +255,18 @@ Int DisconnectManager::getPingsReceived() void DisconnectManager::updateDisconnectStatus(ConnectionManager *conMgr) { + Bool removedPlayer = FALSE; for (Int i = 0; i < MAX_SLOTS; ++i) { if (conMgr->isPlayerConnected(i)) { Int slot = translatedSlotPosition(i, conMgr->getLocalPlayerID()); if (slot != -1) { time_t curTime = timeGetTime(); time_t newTime = TheGlobalData->m_networkPlayerTimeoutTime - (curTime - m_playerTimeouts[slot]); +#if !RETAIL_COMPATIBLE_NETWORKING + if (m_closedGracefully[slot]) { + newTime = -1; + } +#endif // if someone is more than 2/3 timed out, lets get our frame numbers sync'd up. Also if someone is voted out // lets do the same thing. @@ -217,7 +280,11 @@ void DisconnectManager::updateDisconnectStatus(ConnectionManager *conMgr) { DEBUG_LOG(("DisconnectManager::updateDisconnectStatus - curTime = %I64d, m_timeOfDisconnectScreenOn = %I64d, curTime - m_timeOfDisconnectScreenOn = %I64d", (Int64)curTime, (Int64)m_timeOfDisconnectScreenOn, (Int64)(curTime - m_timeOfDisconnectScreenOn))); if (m_timeOfDisconnectScreenOn != 0) { +#if RETAIL_COMPATIBLE_NETWORKING if ((curTime - m_timeOfDisconnectScreenOn) > TheGlobalData->m_networkDisconnectScreenNotifyTime) { +#else + if ((curTime - m_timeOfDisconnectScreenOn) > std::min(TheGlobalData->m_networkDisconnectScreenNotifyTime, SCREEN_NOTIFY_TIME)) { +#endif TheNetwork->notifyOthersOfCurrentFrame(); m_haveNotifiedOtherPlayersOfCurrentFrame = TRUE; } @@ -232,9 +299,8 @@ void DisconnectManager::updateDisconnectStatus(ConnectionManager *conMgr) { if (isLocalPlayerNextPacketRouter(conMgr) == TRUE) { DEBUG_LOG(("DisconnectManager::updateDisconnectStatus - local player is next packet router")); DEBUG_LOG(("DisconnectManager::updateDisconnectStatus - about to do the disconnect procedure for player %d", i)); - sendDisconnectCommand(i, conMgr); - disconnectPlayer(i, conMgr); - sendPlayerDestruct(i, conMgr); + removePlayer(i, conMgr); + removedPlayer = TRUE; } else { DEBUG_LOG(("DisconnectManager::updateDisconnectStatus - local player is not the next packet router")); } @@ -246,6 +312,22 @@ void DisconnectManager::updateDisconnectStatus(ConnectionManager *conMgr) { } } } + + if ((removedPlayer == TRUE) && (allOnSameFrame(conMgr) == TRUE) && (isLocalPlayerNextPacketRouter(conMgr) == TRUE)) { + for (Int i = 0; i < MAX_SLOTS; ++i) { + Int slot = translatedSlotPosition(i, conMgr->getLocalPlayerID()); + if ((slot != -1) && (conMgr->isPlayerConnected(i) == TRUE) && (isPlayerInGame(slot, conMgr) == FALSE)) { + DEBUG_LOG(("DisconnectManager::updateDisconnectStatus - about to do the disconnect procedure for player %d in the same pass", i)); + removePlayer(i, conMgr); + } + } + } +} + +void DisconnectManager::removePlayer(Int slot, ConnectionManager *conMgr) { + sendDisconnectCommand(slot, conMgr); + disconnectPlayer(slot, conMgr); + sendPlayerDestruct(slot, conMgr); } void DisconnectManager::updateWaitForPacketRouter(ConnectionManager *conMgr) { @@ -306,23 +388,143 @@ void DisconnectManager::processDisconnectKeepAlive(NetCommandMsg *msg, Connectio Int slot = translatedSlotPosition(cmdMsg->getPlayerID(), conMgr->getLocalPlayerID()); if (slot != -1) { resetPlayerTimeout(slot); +#if !RETAIL_COMPATIBLE_NETWORKING + m_closedGracefully[slot] = FALSE; +#endif } } +#if !RETAIL_COMPATIBLE_NETWORKING +void DisconnectManager::playerClosedGracefully(Int slot, ConnectionManager *conMgr) { + if (slot < 0 || slot >= MAX_SLOTS) { + return; + } + Int transSlot = translatedSlotPosition(slot, conMgr->getLocalPlayerID()); + if (transSlot == -1 || conMgr->isPlayerConnected(slot) == FALSE) { + return; + } + // The player counts as timed out, so the next packet router removes it with the usual disconnect command. + m_closedGracefully[transSlot] = TRUE; +} +#endif + void DisconnectManager::processDisconnectPlayer(NetCommandMsg *msg, ConnectionManager *conMgr) { NetDisconnectPlayerCommandMsg *cmdMsg = (NetDisconnectPlayerCommandMsg *)msg; DEBUG_LOG(("DisconnectManager::processDisconnectPlayer - Got disconnect player command from player %d. Disconnecting player %d on frame %d", msg->getPlayerID(), cmdMsg->getDisconnectSlot(), cmdMsg->getDisconnectFrame())); - DEBUG_ASSERTCRASH(TheGameLogic->getFrame() == cmdMsg->getDisconnectFrame(), ("disconnecting player on the wrong frame!!!")); - Int slot = cmdMsg->getDisconnectSlot(); + Int slot = cmdMsg->getDisconnectSlot(); + if (slot < 0 || slot >= MAX_SLOTS) { + DEBUG_LOG(("DisconnectManager::processDisconnectPlayer - slot %d is not in the game, ignored", slot)); + return; + } - // if this is a self initiated quit (Quit button on DC menu), - // send a destroy-player command so assets are removed. - if (cmdMsg->getPlayerID() == slot) { - // sendPlayerDestruct assumes we are (or we will become) the router. +#if RETAIL_COMPATIBLE_NETWORKING + // a player that quits from the disconnect screen also destroys its assets + if (cmdMsg->getPlayerID() == (UnsignedInt)slot) { sendPlayerDestruct(slot, conMgr); } +#endif + + if ((slot != (Int)conMgr->getLocalPlayerID()) && (conMgr->isPlayerConnected(slot) == FALSE)) { + DEBUG_LOG(("DisconnectManager::processDisconnectPlayer - player %d is already disconnected", slot)); + return; + } + +#if !RETAIL_COMPATIBLE_NETWORKING + if ((cmdMsg->getPlayerID() == (UnsignedInt)slot) || (cmdMsg->getDisconnectFrame() == LEAVE_NOTICE_FRAME)) { + if (cmdMsg->getPlayerID() != (UnsignedInt)slot) { +#if defined(GENERALS_ONLINE) + if (!conMgr->verifyLeaveNotice(cmdMsg)) { + DEBUG_LOG(("DisconnectManager::processDisconnectPlayer - player %d said that player %d left without its signature, ignored", msg->getPlayerID(), slot)); + return; + } +#else + return; +#endif + } + playerClosedGracefully(slot, conMgr); + + // everyone that got the leave from the leaver tells the others, so all of them pick the same next packet router + if (cmdMsg->getPlayerID() == (UnsignedInt)slot) { + NetDisconnectPlayerCommandMsg *noticeMsg = newInstance(NetDisconnectPlayerCommandMsg); + noticeMsg->setDisconnectSlot(slot); + noticeMsg->setDisconnectFrame(LEAVE_NOTICE_FRAME); + noticeMsg->setPlayerID(conMgr->getLocalPlayerID()); + if (cmdMsg->hasSignature()) { + noticeMsg->setSignature(cmdMsg->getSignature()); // the leaver's proof travels with the notice + } + if (DoesCommandRequireACommandID(noticeMsg->getNetCommandType())) { + noticeMsg->setID(GenerateNextCommandID()); + } + conMgr->sendLocalCommandDirect(noticeMsg, (UnsignedByte)(0xff & ~(1 << conMgr->getLocalPlayerID()) & ~(1 << slot))); + noticeMsg->detach(); + } + return; + } + + // a removal is the call of the acting packet router + if ((Int)cmdMsg->getPlayerID() != getActingPacketRouterSlot(conMgr)) { + DEBUG_LOG(("DisconnectManager::processDisconnectPlayer - player %d is not the acting packet router, removal of player %d ignored", msg->getPlayerID(), slot)); + return; + } +#endif + + DEBUG_ASSERTCRASH(TheGameLogic->getFrame() == cmdMsg->getDisconnectFrame(), ("disconnecting player on the wrong frame!!!")); disconnectPlayer(cmdMsg->getDisconnectSlot(), conMgr); + +#if RETAIL_COMPATIBLE_NETWORKING + if (cmdMsg->getPlayerID() == (UnsignedInt)slot && isLocalPlayerNextPacketRouter(conMgr) == TRUE) { + m_relayFrame[slot] = TheGameLogic->getFrame(); + m_relayTo[slot] = 0; + for (Int i = 0; i < MAX_SLOTS; ++i) { + if ((i != slot) && (i != (Int)conMgr->getLocalPlayerID()) && (conMgr->isPlayerConnected(i) == TRUE)) { + m_relayTo[slot] |= (UnsignedByte)(1 << i); + } + } + } +#endif +} + +#if RETAIL_COMPATIBLE_NETWORKING +void DisconnectManager::sendPendingRelays(ConnectionManager *conMgr) { + for (Int slot = 0; slot < MAX_SLOTS; ++slot) { + if (m_relayTo[slot] == 0) { + continue; + } + if (isLocalPlayerNextPacketRouter(conMgr) == FALSE) { + m_relayTo[slot] = 0; + continue; + } + UnsignedByte sendTo = 0; + for (Int i = 0; i < MAX_SLOTS; ++i) { + const UnsignedByte bit = (UnsignedByte)(1 << i); + if ((m_relayTo[slot] & bit) == 0) { + continue; + } + if (conMgr->isPlayerConnected(i) == FALSE) { + m_relayTo[slot] &= (UnsignedByte)~bit; + } + // pass the leave on to players stalled on the frame it was applied on + else if ((m_disconnectFramesReceived[i] == TRUE) && (m_disconnectFrames[i] == m_relayFrame[slot])) { + sendTo |= bit; + m_relayTo[slot] &= (UnsignedByte)~bit; + } + } + if (sendTo == 0) { + continue; + } + DEBUG_LOG(("DisconnectManager::sendPendingRelays - passing on the leave of player %d on frame %d to mask %X", slot, m_relayFrame[slot], sendTo)); + NetDisconnectPlayerCommandMsg *relayMsg = newInstance(NetDisconnectPlayerCommandMsg); + relayMsg->setDisconnectSlot(slot); + relayMsg->setDisconnectFrame(m_relayFrame[slot]); + relayMsg->setPlayerID(conMgr->getLocalPlayerID()); + if (DoesCommandRequireACommandID(relayMsg->getNetCommandType())) { + relayMsg->setID(GenerateNextCommandID()); + } + conMgr->sendLocalCommand(relayMsg, sendTo); + relayMsg->detach(); + } } +#endif void DisconnectManager::processPacketRouterQuery(NetCommandMsg *msg, ConnectionManager *conMgr) { NetPacketRouterQueryCommandMsg *cmdMsg = (NetPacketRouterQueryCommandMsg *)msg; @@ -462,6 +664,11 @@ void DisconnectManager::nextFrame(UnsignedInt frame, ConnectionManager *conMgr) void DisconnectManager::allCommandsReady(UnsignedInt frame, ConnectionManager *conMgr, Bool waitForPacketRouter) { if (m_disconnectState != DISCONNECTSTATETYPE_SCREENOFF) { DEBUG_LOG(("DisconnectManager::allCommandsReady - setting screen state to off.")); +#if !RETAIL_COMPATIBLE_NETWORKING + // no removal: a stall on the next frame brings the screen back soon + m_catchUpFrame = (m_removedInRound == TRUE) ? 0 : frame + 1; + m_removedInRound = FALSE; +#endif TheDisconnectMenu->hideScreen(); m_disconnectState = DISCONNECTSTATETYPE_SCREENOFF; @@ -474,6 +681,14 @@ void DisconnectManager::allCommandsReady(UnsignedInt frame, ConnectionManager *c DEBUG_LOG(("DisconnectManager::allCommandsReady - resetting m_timeOfDisconnectScreenOn")); m_timeOfDisconnectScreenOn = 0; + +#if !RETAIL_COMPATIBLE_NETWORKING + // the screen stays up until the player that has left is removed + m_screenKept = isGonePlayerConnected(conMgr); + if (m_screenKept == TRUE) { + TheDisconnectMenu->showScreen(); + } +#endif } } @@ -561,7 +776,13 @@ void DisconnectManager::turnOnScreen(ConnectionManager *conMgr) { m_disconnectState = DISCONNECTSTATETYPE_SCREENON; m_lastKeepAliveSendTime = -1; populateDisconnectScreen(conMgr); - resetPlayerTimeouts(conMgr); + // the count-downs run from the stall start + for (Int i = 0; i < MAX_SLOTS; ++i) { + Int slot = translatedSlotPosition(i, conMgr->getLocalPlayerID()); + if ((slot != -1) && (m_playerTimeouts[slot] < m_lastFrameTime)) { + m_playerTimeouts[slot] = m_lastFrameTime; + } + } TheDisconnectMenu->hidePacketRouterTimeout(); m_haveNotifiedOtherPlayersOfCurrentFrame = FALSE; @@ -571,6 +792,9 @@ void DisconnectManager::turnOnScreen(ConnectionManager *conMgr) { } void DisconnectManager::disconnectPlayer(Int slot, ConnectionManager *conMgr) { +#if !RETAIL_COMPATIBLE_NETWORKING + m_removedInRound = TRUE; +#endif DEBUG_LOG(("DisconnectManager::disconnectPlayer - Disconnecting slot number %d on frame %d", slot, TheGameLogic->getFrame())); DEBUG_ASSERTCRASH((slot >= 0) && (slot < MAX_SLOTS), ("Attempting to disconnect an invalid slot number")); if ((slot < 0) || (slot >= (MAX_SLOTS))) { @@ -614,7 +838,7 @@ void DisconnectManager::sendDisconnectCommand(Int slot, ConnectionManager *conMg return; } - UnsignedInt disconnectFrame = getMaxDisconnectFrame(); + UnsignedInt disconnectFrame = getMaxDisconnectFrame(conMgr); // Need to do the NetDisconnectPlayerCommandMsg creation and sending here. NetDisconnectPlayerCommandMsg *msg = newInstance(NetDisconnectPlayerCommandMsg); @@ -691,6 +915,23 @@ Bool DisconnectManager::allOnSameFrame(ConnectionManager *conMgr) { return retval; } +#if !RETAIL_COMPATIBLE_NETWORKING +// The first packet router in line that is still in the game, -1 if there is none. +Int DisconnectManager::getActingPacketRouterSlot(ConnectionManager *conMgr) { + UnsignedInt localSlot = conMgr->getLocalPlayerID(); + UnsignedInt packetRouterSlot = conMgr->getPacketRouterSlot(); + Int transSlot = translatedSlotPosition(packetRouterSlot, localSlot); + while ((transSlot != -1) && (isPlayerInGame(transSlot, conMgr) == FALSE)) { + packetRouterSlot = conMgr->getNextPacketRouterSlot(packetRouterSlot); + if (packetRouterSlot >= MAX_SLOTS) { + return -1; + } + transSlot = translatedSlotPosition(packetRouterSlot, localSlot); + } + return (Int)packetRouterSlot; +} +#endif + Bool DisconnectManager::isLocalPlayerNextPacketRouter(ConnectionManager *conMgr) { UnsignedInt localSlot = conMgr->getLocalPlayerID(); UnsignedInt packetRouterSlot = conMgr->getPacketRouterSlot(); @@ -720,6 +961,12 @@ Bool DisconnectManager::hasPlayerTimedOut(Int slot) { return FALSE; } +#if !RETAIL_COMPATIBLE_NETWORKING + if (m_closedGracefully[slot]) { + return TRUE; + } +#endif + time_t newTime = TheGlobalData->m_networkPlayerTimeoutTime - (timeGetTime() - m_playerTimeouts[slot]); if (newTime <= 0) { return TRUE; @@ -749,6 +996,46 @@ void DisconnectManager::sendPlayerDestruct(Int slot, ConnectionManager *conMgr) netmsg->detach(); } +Bool DisconnectManager::isPlayerSilent(Int slot) { + if ((slot == -1) || (m_disconnectState == DISCONNECTSTATETYPE_SCREENOFF) || (m_timeOfDisconnectScreenOn == 0)) { + return FALSE; + } +#if !RETAIL_COMPATIBLE_NETWORKING + if (m_closedGracefully[slot]) { + return TRUE; + } +#endif + const time_t curTime = timeGetTime(); + // Nobody is silent before the keep alives had time to arrive. + if ((curTime - m_timeOfDisconnectScreenOn) < (time_t)SILENT_PLAYER_TIME) { + return FALSE; + } + return (curTime - m_playerTimeouts[slot]) >= (time_t)SILENT_PLAYER_TIME; +} + +Bool DisconnectManager::isPlayerGone(Int slot, ConnectionManager *conMgr) { + if (isPlayerSilent(slot) == FALSE) { + return FALSE; + } + if (hasPlayerTimedOut(slot) == TRUE) { + return TRUE; + } + const Int localSlot = conMgr->getLocalPlayerID(); + const Int transSlot = untranslatedSlotPosition(slot, localSlot); + for (Int i = 0; i < MAX_SLOTS; ++i) { + if ((i == transSlot) || (conMgr->isPlayerConnected(i) == FALSE)) { + continue; + } + if ((i != localSlot) && (isPlayerSilent(translatedSlotPosition(i, localSlot)) == TRUE)) { + continue; + } + if ((m_playerVotes[transSlot][i].vote == FALSE) || (m_playerVotes[transSlot][i].frame != TheGameLogic->getFrame())) { + return FALSE; + } + } + return TRUE; +} + // the 'slot' variable is supposed to be a translated slot position. (translated slot meaning // that it is the player's position in the disconnect menu) Bool DisconnectManager::isPlayerVotedOut(Int slot, ConnectionManager *conMgr) { @@ -756,17 +1043,41 @@ Bool DisconnectManager::isPlayerVotedOut(Int slot, ConnectionManager *conMgr) { // we can't vote out ourselves. return FALSE; } - Int transSlot = untranslatedSlotPosition(slot, conMgr->getLocalPlayerID()); + const Int localSlot = conMgr->getLocalPlayerID(); + Int transSlot = untranslatedSlotPosition(slot, localSlot); Int numVotes = countVotesForPlayer(transSlot); - if (numVotes >= (conMgr->getNumPlayers() - 1)) { - return TRUE; + Int numVoters = conMgr->getNumPlayers() - 1; + + // a player that is gone cannot vote, neither its vote nor its missing vote counts + for (Int i = 0; i < MAX_SLOTS; ++i) { + if ((i == transSlot) || (i == localSlot) || (conMgr->isPlayerConnected(i) == FALSE)) { + continue; + } + if (isPlayerGone(translatedSlotPosition(i, localSlot), conMgr) == TRUE) { + --numVoters; + if ((m_playerVotes[transSlot][i].vote == TRUE) && (m_playerVotes[transSlot][i].frame == TheGameLogic->getFrame())) { + --numVotes; + } + } } - return FALSE; + + // One vote alone never removes a player of a game with more than two. + if ((numVoters < 2) && (conMgr->getNumPlayers() > 2)) { + numVoters = 2; + } + return numVotes >= numVoters; } -UnsignedInt DisconnectManager::getMaxDisconnectFrame() { +UnsignedInt DisconnectManager::getMaxDisconnectFrame(ConnectionManager *conMgr) { UnsignedInt retval = 0; for (Int i = 0; i < MAX_SLOTS; ++i) { +#if !RETAIL_COMPATIBLE_NETWORKING + // a player that has left reports the frame it stopped on, which the others may never reach + Int transSlot = translatedSlotPosition(i, conMgr->getLocalPlayerID()); + if ((transSlot != -1) && (m_closedGracefully[transSlot] == TRUE)) { + continue; + } +#endif if (m_disconnectFrames[i] > retval) { retval = m_disconnectFrames[i]; } @@ -774,6 +1085,32 @@ UnsignedInt DisconnectManager::getMaxDisconnectFrame() { return retval; } +#if !RETAIL_COMPATIBLE_NETWORKING +Bool DisconnectManager::isGonePlayerConnected(ConnectionManager *conMgr) { + for (Int i = 0; i < MAX_SLOTS; ++i) { + Int transSlot = translatedSlotPosition(i, conMgr->getLocalPlayerID()); + if ((transSlot != -1) && (m_closedGracefully[transSlot] == TRUE) && (conMgr->isPlayerConnected(i) == TRUE)) { + return TRUE; + } + } + return FALSE; +} +#endif + +#if !RETAIL_COMPATIBLE_NETWORKING +Bool DisconnectManager::isPlayerOutOfGame(Int slot, ConnectionManager *conMgr) { + if ((slot < 0) || (slot >= MAX_SLOTS) || (conMgr->isPlayerConnected(slot) == FALSE)) { + return FALSE; + } + Int transSlot = translatedSlotPosition(slot, conMgr->getLocalPlayerID()); + if (transSlot == -1) { + return FALSE; + } + return (isPlayerInGame(transSlot, conMgr) == FALSE); +} + +#endif + Bool DisconnectManager::isPlayerInGame(Int slot, ConnectionManager *conMgr) { Int transSlot = untranslatedSlotPosition(slot, conMgr->getLocalPlayerID()); DEBUG_ASSERTCRASH((transSlot >= 0) && (transSlot < MAX_SLOTS), ("invalid slot number")); diff --git a/Core/GameEngine/Source/GameNetwork/NetPacket.cpp b/Core/GameEngine/Source/GameNetwork/NetPacket.cpp index 3a76de3b860..22c43ce187d 100644 --- a/Core/GameEngine/Source/GameNetwork/NetPacket.cpp +++ b/Core/GameEngine/Source/GameNetwork/NetPacket.cpp @@ -323,6 +323,9 @@ Bool NetPacket::isFrameRepeat(NetCommandRef *msg) { } NetFrameCommandMsg *framemsg = (NetFrameCommandMsg *)(msg->getCommand()); NetFrameCommandMsg *lastmsg = (NetFrameCommandMsg *)(m_lastCommand->getCommand()); + if (framemsg->getPlayerID() != lastmsg->getPlayerID()) { + return FALSE; + } if (framemsg->getCommandCount() != 0) { return FALSE; } diff --git a/Core/GameEngine/Source/GameNetwork/Network.cpp b/Core/GameEngine/Source/GameNetwork/Network.cpp index ea6d13f9f83..9f321d53b54 100644 --- a/Core/GameEngine/Source/GameNetwork/Network.cpp +++ b/Core/GameEngine/Source/GameNetwork/Network.cpp @@ -607,6 +607,13 @@ Bool Network::processCommand(GameMessage* msg) m_lastFrame = TheGameLogic->getFrame(); } +#if defined(GENERALS_ONLINE) + if (msg->getType() == GameMessage::MSG_CLEAR_GAME_DATA && + (m_localStatus == NETLOCALSTATUS_LEAVING || m_localStatus == NETLOCALSTATUS_LEFT)) { + return TRUE; // the exit is already under way + } +#endif + // Are we leaving the game? // This has to happen after the check to see if this is the start of a new logic frame. // The reason is that we have to send all the frame info packets necessary to get to the diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h index 489c97b6dfe..0b69979da0a 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h @@ -50,6 +50,8 @@ class PlayerConnection EConnectionState GetState() const { return m_State; } int SendGamePacket(void* pBuffer, uint32_t totalDataSize); + bool LogInvalidDue(); ///< true about every 5 s while the connection stays invalid, the rest of its lines stay out of the log + int64_t m_nextInvalidLogMs = 0; void SendACPacket(const void* pData, uint32_t dataLen); diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NextGenTransport.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NextGenTransport.h index 71814f94ee7..315d2bd64df 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NextGenTransport.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NextGenTransport.h @@ -49,6 +49,9 @@ class NextGenTransport : public Transport //: public MemoryPoolObject } private: + Bool sendFailureLogDue(); ///< true once a second, the other failure lines are counted + UnsignedInt m_nextSendFailLogMs = 0; + Int m_skippedSendFailLogs = 0; OutgoingPacketState m_outPacketState[MAX_MESSAGES]; // Track which incoming buffer slots are occupied (handles zero-length packets) bool m_inBufferOccupied[MAX_MESSAGES]; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index 1277851dc6d..3a0681e309d 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -348,8 +348,10 @@ struct ServiceConfig float ibra_maxslack_greaterthan200ms = 1.f; using Send = NetworkSettings::Send; using InputDelay = NetworkSettings::InputDelay; + using Disconnect = NetworkSettings::Disconnect; Send send; InputDelay input_delay; + Disconnect disconnect; struct Lobby { @@ -455,6 +457,12 @@ inline void ReadServiceConfig(const nlohmann::json& j, ServiceConfig& c) ReadServiceConfigValue(f, "latency_history_samples", c.input_delay.formula.latency_history_samples); } } + if (j.contains("disconnect")) + { + const nlohmann::json& s = j["disconnect"]; + ReadServiceConfigValue(s, "freeze_before_screen_ms", c.disconnect.freeze_before_screen_ms); + ReadServiceConfigValue(s, "remove_after_ms", c.disconnect.remove_after_ms); + } if (j.contains("lobby")) { ReadServiceConfigValue(j["lobby"], "host_migration", c.lobby.host_migration); @@ -488,6 +496,9 @@ inline void ClampServiceConfig(ServiceConfig& c) c.input_delay.formula.latency_history_samples = std::clamp(c.input_delay.formula.latency_history_samples, 128, 3600); // FrameMetrics drops pending acks when the window is smaller than the frames in flight c.send.frames_per_packet = std::clamp(c.send.frames_per_packet, 1, 16); + c.disconnect.freeze_before_screen_ms = std::clamp(c.disconnect.freeze_before_screen_ms, 1000, 30000); + c.disconnect.remove_after_ms = std::clamp(c.disconnect.remove_after_ms, 10000, 180000); + c.telemetry.screenshot_width = std::clamp(c.telemetry.screenshot_width, 16, 4096); c.telemetry.screenshot_height = std::clamp(c.telemetry.screenshot_height, 16, 4096); } diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp index f42004617eb..c4341e7d817 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp @@ -128,9 +128,20 @@ void OnSteamNetConnectionStatusChanged(SteamNetConnectionStatusChangedCallback_t const int64_t userID = plrConnection.m_userID; const int signallingAttemptsBeforeDisconnect = plrConnection.m_SignallingAttempts; + const bool bInMatch = TheGameLogic != nullptr && TheGameLogic->isInInternetGame(); if (TheNetwork != nullptr) { TheNetwork->GetConnectionManager()->disconnectPlayer(userID); + + // a peer that closed on purpose during the match has left; closing to negotiate a new connection is not a leave + const bool bClosedOnPurpose = pInfo->m_info.m_eState == k_ESteamNetworkingConnectionState_ClosedByPeer + && pInfo->m_info.m_eEndReason == k_ESteamNetConnectionEnd_App_Generic + && strstr(pInfo->m_info.m_szEndDebug, "new connection being negotiated") == nullptr + && strstr(pInfo->m_info.m_szEndDebug, "(Rejected)") == nullptr; + if (bClosedOnPurpose && bInMatch) + { + TheNetwork->GetConnectionManager()->playerClosedGracefully(userID); + } } NetworkLog(ELogVerbosity::LOG_RELEASE, "[DC] Closing connection %lld", userID); @@ -143,7 +154,6 @@ void OnSteamNetConnectionStatusChanged(SteamNetConnectionStatusChangedCallback_t const bool bWeJoinedLater = pJoinOrderLobby == nullptr || !pJoinOrderLobby->IsJoinOrderKnown() || pJoinOrderLobby->JoinedAfter(userID); const bool bPeerLeft = pJoinOrderLobby != nullptr && pJoinOrderLobby->IsJoinOrderKnown() && !pJoinOrderLobby->IsLobbyMember(userID); // a match can't leave its lobby, so keep repairing the link until the service drops the player - const bool bInMatch = TheGameLogic != nullptr && TheGameLogic->isInInternetGame(); bool bShouldRetry = serviceConf.connection.retry_signalling && ((bInMatch && !bPeerLeft) || (!bWeJoinedLater && !bPeerLeft) || signallingAttemptsBeforeDisconnect < numSignallingAttempts); bool bWasError = pInfo->m_info.m_eState == k_ESteamNetworkingConnectionState_ProblemDetectedLocally || pInfo->m_info.m_eEndReason != k_ESteamNetConnectionEnd_App_Generic; @@ -1802,6 +1812,17 @@ PlayerConnection::PlayerConnection(int64_t userID, const char* szMiddlewareID) } } +bool PlayerConnection::LogInvalidDue() +{ + const int64_t nowMs = NowSteadyMs(); + if (nowMs < m_nextInvalidLogMs) + { + return false; + } + m_nextInvalidLogMs = nowMs + 5000; + return true; +} + int PlayerConnection::SendGamePacket(void* pBuffer, uint32_t totalDataSize) { if (totalDataSize == 0) @@ -1825,7 +1846,10 @@ int PlayerConnection::SendGamePacket(void* pBuffer, uint32_t totalDataSize) { if (m_hSteamConnection == k_HSteamNetConnection_Invalid) { - NetworkLog(ELogVerbosity::LOG_RELEASE, "[GAME PACKET] Cannot send game packet - connection is invalid for user %lld", m_userID); + if (LogInvalidDue()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[GAME PACKET] Cannot send game packet - connection is invalid for user %lld", m_userID); + } return -(int)k_EResultFail; } @@ -1953,7 +1977,10 @@ int PlayerConnection::Recv(SteamNetworkingMessage_t** pMsg, int maxMessages) } else { - NetworkLog(ELogVerbosity::LOG_RELEASE, "[DISC] Recv Failed 1 from user %lld", m_userID); + if (LogInvalidDue()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[DISC] Recv Failed 1 from user %lld", m_userID); + } } return r; diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp index 404094f7c7d..8c387770c45 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp @@ -634,6 +634,18 @@ Bool NextGenTransport::doRecv(void) return bRet; } +Bool NextGenTransport::sendFailureLogDue() +{ + const UnsignedInt nowMs = timeGetTime(); + if (static_cast(nowMs - m_nextSendFailLogMs) < 0) + { + ++m_skippedSendFailLogs; + return FALSE; + } + m_nextSendFailLogMs = nowMs + 1000; + return TRUE; +} + Bool NextGenTransport::doSend(void) { Bool retval = TRUE; @@ -742,19 +754,27 @@ Bool NextGenTransport::doSend(void) if (m_outPacketState[i].retryCount < OutgoingPacketState::MAX_RETRIES) { - NetworkLog(ELogVerbosity::LOG_RELEASE, - "Game Packet Send: SendGamePacket failed (err=%d), retry %d/%d for packet to user %lld", - sendResult, m_outPacketState[i].retryCount, - OutgoingPacketState::MAX_RETRIES, pSlot->m_userID); + if (sendFailureLogDue()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, + "Game Packet Send: SendGamePacket failed (err=%d), retry %d/%d for packet to user %lld, %d lines skipped", + sendResult, m_outPacketState[i].retryCount, + OutgoingPacketState::MAX_RETRIES, pSlot->m_userID, m_skippedSendFailLogs); + m_skippedSendFailLogs = 0; + } // Keep packet in queue for retry retval = FALSE; } else { // Max retries exceeded - drop packet - NetworkLog(ELogVerbosity::LOG_RELEASE, - "Game Packet Send: Dropping packet after %d failed retries to user %lld", - m_outPacketState[i].retryCount, pSlot->m_userID); + if (sendFailureLogDue()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, + "Game Packet Send: Dropping packet after %d failed retries to user %lld, %d lines skipped", + m_outPacketState[i].retryCount, pSlot->m_userID, m_skippedSendFailLogs); + m_skippedSendFailLogs = 0; + } m_outBuffer[i].length = 0; m_outPacketState[i].retryCount = 0; retval = FALSE; @@ -850,7 +870,11 @@ Bool NextGenTransport::queueSend(UnsignedInt addr, return TRUE; } - NetworkLog(ELogVerbosity::LOG_RELEASE, - "Game Packet QueueSend: m_outBuffer full, dropping packet"); + if (sendFailureLogDue()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, + "Game Packet QueueSend: m_outBuffer full, dropping packet, %d lines skipped", m_skippedSendFailLogs); + m_skippedSendFailLogs = 0; + } return FALSE; } From ecdf4f992747d2c5361c2da8d521ab70eaeeb267 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Thu, 22 Oct 2026 07:03:29 +0200 Subject: [PATCH 10/12] feat(network): Revise runahead from measured latency and frame pacing --- .../Include/GameNetwork/Connection.h | 2 + .../Include/GameNetwork/ConnectionManager.h | 14 + .../Include/GameNetwork/FrameMetrics.h | 2 +- .../Include/GameNetwork/NetworkInterface.h | 1 + .../Include/GameNetwork/NetworkSettings.h | 11 +- .../Source/GameNetwork/Connection.cpp | 15 + .../Source/GameNetwork/ConnectionManager.cpp | 286 ++++++++---------- .../Source/GameNetwork/FrameMetrics.cpp | 25 +- .../Source/GameNetwork/GameSpy/LobbyUtils.cpp | 2 +- .../GameEngine/Source/GameNetwork/Network.cpp | 46 ++- .../GameEngine/Include/GameClient/InGameUI.h | 1 + .../GameNetwork/GeneralsOnline/NGMP_include.h | 1 - .../GameNetwork/GeneralsOnline/NetworkMesh.h | 47 +-- .../GeneralsOnline/OnlineServices_Init.h | 28 +- .../GameEngine/Source/Common/GlobalData.cpp | 4 + .../GameEngine/Source/GameClient/InGameUI.cpp | 34 +-- .../GameNetwork/GeneralsOnline/NGMPGame.cpp | 3 +- .../GeneralsOnline/NGMP_Helpers.cpp | 5 - .../GeneralsOnline/NetworkMesh.cpp | 54 +--- 19 files changed, 286 insertions(+), 295 deletions(-) diff --git a/Core/GameEngine/Include/GameNetwork/Connection.h b/Core/GameEngine/Include/GameNetwork/Connection.h index c1f409e3c7e..a0425fe0522 100644 --- a/Core/GameEngine/Include/GameNetwork/Connection.h +++ b/Core/GameEngine/Include/GameNetwork/Connection.h @@ -68,6 +68,7 @@ class Connection : public MemoryPoolObject void setNetworkSettings(const NetworkSettings &settings, Bool unreliableTransport) { m_networkSettings = settings; m_unreliableTransport = unreliableTransport; } #endif void setSlot(Int slot) { m_slot = slot; } + void setShortGrouping(Bool shortGrouping); ///< Send at one frame while the run ahead is close to its floor. Int getRedundancyDepth() const { return m_redundancyDepth; } time_t getFrameGrouping() const { return m_frameGrouping; } ///< The send interval in use, shorter than the configured one while redundancy is on. @@ -123,6 +124,7 @@ class Connection : public MemoryPoolObject Bool m_queueSuspected; ///< A full queue on the path, redundancy steps down instead of up. Int m_slot; + Bool m_shortGrouping; time_t m_configuredFrameGrouping; time_t m_frameGrouping; ///< The minimum time between packet sends. time_t m_lastTimeSent; ///< The time of the last packet send. diff --git a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h index dd2268297e1..667ff4f445e 100644 --- a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h +++ b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h @@ -263,6 +263,8 @@ class ConnectionManager }; RecentCommandIds m_recentIds[MAX_SLOTS]; + Int m_runAheadLowerCount; + Int m_runAheadLowerPeak; Bool isDuplicateCommand(NetCommandMsg *msg); Int slotForUserID(Int64 userID) const; @@ -352,6 +354,18 @@ class ConnectionManager time_t m_stallStartTime; UnsignedInt m_stallMissingMask; Int m_lastKnownRunAhead; + Bool m_runAheadStillInitial; ///< No run ahead change has taken effect yet, the first decisions apply without the decrease hold. + Int m_pendingRunAhead; ///< Router: the run ahead of the last command it issued. + UnsignedInt m_pendingExecFrame; ///< Router: the execution frame of that command. + Int applyDecreaseHold(Int oldRunAhead, Int askedRunAhead, Int decreaseAfter); +public: + void noteRunAheadApplied(Int oldRunAhead, Int newRunAhead); ///< The run ahead command took effect on this frame. +private: + + Bool m_shortGrouping; ///< The run ahead is within 2 frames of its floor, connections send at one frame + Int getRunAheadFloor() const; + time_t getEffectiveDecisionIntervalMs(Int runAhead) const; ///< The longer of the configured floor and 2 x the run ahead in ms + void updateShortGrouping(); Int m_frameGroupingMs; public: Bool getOverlayInfo(Int slot, NetworkOverlayInfo &info); ///< Fills what the overlay and the connection rating need about one slot. diff --git a/Core/GameEngine/Include/GameNetwork/FrameMetrics.h b/Core/GameEngine/Include/GameNetwork/FrameMetrics.h index f717b2ba03d..90a85df2db6 100644 --- a/Core/GameEngine/Include/GameNetwork/FrameMetrics.h +++ b/Core/GameEngine/Include/GameNetwork/FrameMetrics.h @@ -48,7 +48,7 @@ class FrameMetrics { void addCushion(Int cushion); Real getAverageLatency(); - Int getAverageFPS(); + Int getAverageFPS(); ///< The median of the per-second fps samples. Int getMinimumCushion(); Bool isPending(UnsignedInt frame); diff --git a/Core/GameEngine/Include/GameNetwork/NetworkInterface.h b/Core/GameEngine/Include/GameNetwork/NetworkInterface.h index a63e2f5db19..ae327473c3c 100644 --- a/Core/GameEngine/Include/GameNetwork/NetworkInterface.h +++ b/Core/GameEngine/Include/GameNetwork/NetworkInterface.h @@ -62,6 +62,7 @@ class NetworkInterface : public SubsystemInterface virtual Bool isFrameDataReady() = 0; ///< Are the commands for the next frame available? #if defined(GENERALS_ONLINE) virtual Bool isGameExitPending() const { return FALSE; } + virtual UnsignedInt getMeasuredLogicRate() = 0; ///< Logic frames executed per second, over about a second #endif virtual Bool isStalling() = 0; virtual void parseUserList(const GameInfo* game) = 0; ///< Parse a userlist, creating connections diff --git a/Core/GameEngine/Include/GameNetwork/NetworkSettings.h b/Core/GameEngine/Include/GameNetwork/NetworkSettings.h index 32342b7b7ab..a390070d702 100644 --- a/Core/GameEngine/Include/GameNetwork/NetworkSettings.h +++ b/Core/GameEngine/Include/GameNetwork/NetworkSettings.h @@ -23,14 +23,17 @@ struct NetworkSettings } send; struct InputDelay { - int min_frames = 4; - int decision_interval_ms = 5000; - int fps_history_samples = 30; // per-second FPS samples averaged + int min_frames = 0; // lowest run ahead, 0 = send interval frames + 3 + int decision_interval_ms = 1000; + int fps_history_samples = 30; // median of this many per-second FPS samples + int catch_up_frames = 1; // a frame later than this many frame times resets the clock, retail catches up from 2 int slack_percent = 30; // percent added to the measured round trip, also sets the start run ahead and the rating struct Formula { - int latency_history_samples = 200; // in-band ping samples averaged + int latency_window_ms = 0; // in-band round trip averaging window in ms, 0 = since the last decision + int decrease_after_decisions = 20; // decisions in a row that must ask for less before the run ahead falls; 0 = follow at once (retail) + bool repair_budget = true; // add a send interval while a connection repeats commands } formula; } input_delay; diff --git a/Core/GameEngine/Source/GameNetwork/Connection.cpp b/Core/GameEngine/Source/GameNetwork/Connection.cpp index adf1c22d6bd..6bb74a3e6e6 100644 --- a/Core/GameEngine/Source/GameNetwork/Connection.cpp +++ b/Core/GameEngine/Source/GameNetwork/Connection.cpp @@ -57,6 +57,7 @@ Connection::Connection() { m_lastTimeSent = 0; m_frameGrouping = 1; m_configuredFrameGrouping = 1; + m_shortGrouping = FALSE; m_isQuitting = false; m_quitTime = 0; resetRedundancy(); @@ -96,6 +97,7 @@ void Connection::init() { m_lastTimeSent = 0; m_frameGrouping = 1; m_configuredFrameGrouping = 1; + m_shortGrouping = FALSE; m_stats.reset(); resetRedundancy(); m_isQuitting = FALSE; @@ -306,6 +308,12 @@ void Connection::applyFrameGrouping() { } } } + if (m_shortGrouping) { + const time_t oneFrame = (time_t)(1000 / GENERALS_ONLINE_HIGH_FPS_LIMIT); + if (oneFrame < grouping) { + grouping = oneFrame; + } + } #endif m_frameGrouping = grouping; } @@ -739,6 +747,13 @@ NetCommandRef * Connection::processAck(UnsignedShort commandID, UnsignedByte ori return temp; } +void Connection::setShortGrouping(Bool shortGrouping) { + if (shortGrouping != m_shortGrouping) { + m_shortGrouping = shortGrouping; + applyFrameGrouping(); + } +} + void Connection::setFrameGrouping(time_t frameGrouping) { m_configuredFrameGrouping = frameGrouping; applyFrameGrouping(); diff --git a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp index 98dc13e4c4b..c6e4a0aadf1 100644 --- a/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp +++ b/Core/GameEngine/Source/GameNetwork/ConnectionManager.cpp @@ -298,7 +298,7 @@ void ConnectionManager::init() } const NetworkSettings::InputDelay &runAhead = m_networkSettings.input_delay; TheWritableGlobalData->m_networkFPSHistoryLength = (std::max)(1, runAhead.fps_history_samples); - TheWritableGlobalData->m_networkLatencyHistoryLength = (std::max)(1, runAhead.formula.latency_history_samples); + TheWritableGlobalData->m_networkLatencyHistoryLength = (std::max)(30, (runAhead.formula.latency_window_ms > 0 ? runAhead.formula.latency_window_ms : runAhead.decision_interval_ms) * GENERALS_ONLINE_HIGH_FPS_LIMIT / 1000); TheWritableGlobalData->m_networkRunAheadSlack = runAhead.slack_percent; TheWritableGlobalData->m_networkRunAheadMetricsTime = runAhead.decision_interval_ms; MIN_RUNAHEAD = runAhead.min_frames; @@ -366,6 +366,11 @@ void ConnectionManager::init() m_stallStartTime = 0; m_stallMissingMask = 0; m_lastKnownRunAhead = 0; + m_runAheadLowerCount = 0; + m_runAheadLowerPeak = 0; + m_runAheadStillInitial = TRUE; + m_pendingRunAhead = 0; + m_pendingExecFrame = 0; m_loggedSenderUnavailable = FALSE; m_securityDrops = 0; m_directSendActive = FALSE; @@ -373,6 +378,7 @@ void ConnectionManager::init() m_formerRouterMask = 0; m_directSendCheckedAt = 0; clearSignatureState(TRUE); + m_shortGrouping = FALSE; m_frameGroupingMs = 0; #endif @@ -600,6 +606,46 @@ void ConnectionManager::onDirectFrameAck(UnsignedInt frame, Int peerSlot) { } } +Int ConnectionManager::getRunAheadFloor() const { + const NetworkSettings::InputDelay &runAhead = m_networkSettings.input_delay; + if (runAhead.min_frames > 0) { + return runAhead.min_frames; + } + // send interval + receive + execute + margin frame + return (m_shortGrouping ? 1 : m_networkSettings.send.frames_per_packet) + 3; +} + +time_t ConnectionManager::getEffectiveDecisionIntervalMs(Int runAhead) const { + time_t interval = TheGlobalData->m_networkRunAheadMetricsTime; // the floor + const time_t twiceRunAhead = (time_t)(2 * runAhead * 1000 / GENERALS_ONLINE_HIGH_FPS_LIMIT); + if (twiceRunAhead > interval) { + interval = twiceRunAhead; + } + return interval; +} + +// leaving the one frame sends needs 2 frames more than entering them +void ConnectionManager::updateShortGrouping() { + if (TheNetwork == nullptr) { + return; + } + + const Int runAhead = (Int)TheNetwork->getRunAhead(); + const NetworkSettings::InputDelay &config = m_networkSettings.input_delay; + const Int floorOneFrame = (config.min_frames > 0) ? config.min_frames : 4; + if (!m_shortGrouping && runAhead <= floorOneFrame + 2) { + m_shortGrouping = TRUE; + } else if (m_shortGrouping && runAhead > floorOneFrame + 4) { + m_shortGrouping = FALSE; + } + + for (Int i = 0; i < MAX_SLOTS; ++i) { + if (m_connections[i] != nullptr) { + m_connections[i]->setShortGrouping(m_shortGrouping); + } + } +} + Int ConnectionManager::slotForUserID(Int64 userID) const { return (TheNGMPGame != nullptr) ? TheNGMPGame->getSlotOfUser(userID) : -1; } @@ -2245,6 +2291,9 @@ void ConnectionManager::update(Bool isInGame) { } } +#if defined(GENERALS_ONLINE) + updateShortGrouping(); +#endif m_transport->doSend(); } @@ -2256,7 +2305,19 @@ void ConnectionManager::updateRunAhead(Int oldRunAhead, Int frameRate, Bool didS m_lastKnownRunAhead = oldRunAhead; #endif - if ((lasttimesent == 0) || ((curTime - lasttimesent) > TheGlobalData->m_networkRunAheadMetricsTime)) { +#if defined(GENERALS_ONLINE) + const time_t decisionInterval = getEffectiveDecisionIntervalMs(oldRunAhead); +#else + const time_t decisionInterval = TheGlobalData->m_networkRunAheadMetricsTime; +#endif + if ((lasttimesent == 0) || ((curTime - lasttimesent) > decisionInterval)) { +#if defined(GENERALS_ONLINE) + if (m_networkSettings.input_delay.formula.latency_window_ms <= 0) { + // the in-band latency is averaged over the samples since the previous decision + const UnsignedInt samples = (UnsignedInt)(decisionInterval * GENERALS_ONLINE_HIGH_FPS_LIMIT / 1000); + TheWritableGlobalData->m_networkLatencyHistoryLength = (samples > 30) ? samples : 30; + } +#endif if (m_localSlot == m_packetRouterSlot) { // We are the packet router, time to compute a new run ahead for this game. m_latencyAverages[m_localSlot] = m_frameMetrics.getAverageLatency(); @@ -2291,100 +2352,40 @@ void ConnectionManager::updateRunAhead(Int oldRunAhead, Int frameRate, Bool didS // The runahead should always be rounded up to the next integer value to prevent variations in latency from causing stutter // The network slack pushes the runahead up to the next value when the latency is within the slack percentage of the current runahead - Real configuredSlack = (Real)TheGlobalData->m_networkRunAheadSlack / 100.0f; - - Real effectiveSlack = configuredSlack; - -#if defined(GENERALS_ONLINE) - if (TheNGMPGame != nullptr) - { - ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - if (serviceConf.ibra_ra_tweaks) - { - // dynamic slack based on jitter - NetworkMesh* pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); - if (pMesh != nullptr) - { - // 1) Current max latency (RTT) - int maxLatMs = pMesh->getMaximumLatency(); - - // 2) Compute worst-case jitter (max - min) across recent history - int jitterMs = 0; - - auto& connections = pMesh->GetAllConnections(); - for (auto& kvPair : connections) - { - PlayerConnection& conn = kvPair.second; - - if (conn.m_vecLatencyHistory.size() > 1) - { - int minL = INT_MAX; - int maxL = 0; - - for (int l : conn.m_vecLatencyHistory) - { - if (l < minL) minL = l; - if (l > maxL) maxL = l; - } - - int span = maxL - minL; - if (span > jitterMs) - { - jitterMs = span; - } - } - } - - // 3) Convert jitter to a 0?1+ ratio relative to latency - Real jitterRatio = 0.0f; - if (maxLatMs > 0) - { - jitterRatio = (Real)jitterMs / (Real)maxLatMs; - } - - // 4) Map jitter ratio into a slack range based on latency - // Lower latency: tighter slack - // Higher latency: allow more slack to hide jitter - Real minSlack = serviceConf.ibra_minslack_default; - Real maxSlack = serviceConf.ibra_maxslack_default; - - if (maxLatMs > 300) - { - // At high RTT, allow more headroom - minSlack = serviceConf.ibra_minslack_greaterthan300ms; - maxSlack = serviceConf.ibra_maxslack_greaterthan300ms; - } - else if (maxLatMs > 200) - { - // Medium-high latency (200?300 ms) - minSlack = serviceConf.ibra_minslack_greaterthan200ms; - maxSlack = serviceConf.ibra_maxslack_greaterthan200ms; - } - - // Clamp jitterRatio to [0,1] when mapping - if (jitterRatio < 0.0f) jitterRatio = 0.0f; - if (jitterRatio > 1.0f) jitterRatio = 1.0f; - - Real dynamicSlack = minSlack + (maxSlack - minSlack) * jitterRatio; - - // ignore service slack and use our own dynamic slack - effectiveSlack = dynamicSlack; - } - } - } -#endif - - const Real runAheadSlackScale = 1.0f + effectiveSlack; + const Real runAheadSlackScale = 1.0f + (Real)TheGlobalData->m_networkRunAheadSlack / 100.0f; - //NetworkLog(ELogVerbosity::LOG_DEBUG, "RA CALC: %f * %f * %f = %d", getMaximumLatency(), runAheadSlackScale, (Real)minFps, (int)ceilf(getMaximumLatency() * runAheadSlackScale * (Real)minFps)); Int newRunAhead = ceilf(getMaximumLatency() * runAheadSlackScale * (Real)minFps); // TheSuperHackers @info if the runahead goes below 3 logic frames it can start to introduce stutter // We also limit the upper range of the runahead to prevent it getting out of hand Int minRunAheadForClamp = MIN_RUNAHEAD; +#if defined(GENERALS_ONLINE) + minRunAheadForClamp = getRunAheadFloor(); +#endif +#if defined(GENERALS_ONLINE) + // a lost packet is repaired by the copy in the next one, a send interval later: budget for it + Int repairFrames = 0; + if (m_networkSettings.input_delay.formula.repair_budget) { + time_t repairInterval = 0; + for (Int repairSlot = 0; repairSlot < MAX_SLOTS; ++repairSlot) { + if (m_connections[repairSlot] != nullptr && m_connections[repairSlot]->getRedundancyDepth() > 0 && m_connections[repairSlot]->getFrameGrouping() > repairInterval) { + repairInterval = m_connections[repairSlot]->getFrameGrouping(); + } + } + repairFrames = (Int)((repairInterval * GENERALS_ONLINE_HIGH_FPS_LIMIT + 999) / 1000); + } + newRunAhead += repairFrames; +#endif newRunAhead = clamp(minRunAheadForClamp, newRunAhead, MAX_FRAMES_AHEAD / 2); +#if defined(GENERALS_ONLINE) + const Int decreaseAfter = m_networkSettings.input_delay.formula.decrease_after_decisions; + // a command that was issued but has not taken effect yet is the run ahead to decide from, not the one still running + const Int baseRunAhead = (m_pendingExecFrame != 0 && oldRunAhead != m_pendingRunAhead && TheGameLogic->getFrame() <= m_pendingExecFrame + 2) ? m_pendingRunAhead : oldRunAhead; + newRunAhead = applyDecreaseHold(baseRunAhead, newRunAhead, decreaseAfter); +#endif + NetRunAheadCommandMsg* msg = newInstance(NetRunAheadCommandMsg); msg->setPlayerID(m_localSlot); if (DoesCommandRequireACommandID(msg->getNetCommandType())) { @@ -2424,6 +2425,11 @@ void ConnectionManager::updateRunAhead(Int oldRunAhead, Int frameRate, Bool didS msg->setRunAhead(newRunAhead); msg->setFrameRate(minFps); +#if defined(GENERALS_ONLINE) + m_pendingRunAhead = newRunAhead; + m_pendingExecFrame = msg->getExecutionFrame(); +#endif + //DEBUG_LOG(("ConnectionManager::updateRunAhead - new run ahead = %d, new frame rate = %d, execution frame %d", newRunAhead, minFps, msg->getExecutionFrame())); sendLocalCommand(msg, 0xff ^ (1 << minFpsPlayer)); // Send the packet to everyone but the lowest FPS player. @@ -2510,90 +2516,56 @@ void ConnectionManager::updateRunAhead(Int oldRunAhead, Int frameRate, Bool didS } } -Real ConnectionManager::getMaximumLatency() -{ - int latencyLogicModel = 0; - - if (TheNGMPGame != nullptr) - { - ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - latencyLogicModel = serviceConf.network_latency_logic_model; - // 0 = original - // 1 = pick highest between original and Valve latency (current) - // 2 = pick highest between original and Valve latency (historic) - // 3 = use Valve latency (current) - // 4 = use Valve latency (historic) +#if defined(GENERALS_ONLINE) +// The run ahead goes up at once and falls only after decreaseAfter asks for less in a row, then to the highest of them. +Int ConnectionManager::applyDecreaseHold(Int oldRunAhead, Int askedRunAhead, Int decreaseAfter) { + if (m_runAheadStillInitial) { + m_runAheadLowerCount = 0; + return askedRunAhead; } - - Real maxLatency = 0.0f; - - for (Int i = 0; i < MAX_SLOTS; ++i) { - if (isPlayerConnected(i) && m_latencyAverages[i] > maxLatency) { - maxLatency = m_latencyAverages[i]; + if (decreaseAfter > 0 && askedRunAhead < oldRunAhead) { + m_runAheadLowerPeak = (m_runAheadLowerCount == 0) ? askedRunAhead : max(m_runAheadLowerPeak, askedRunAhead); + ++m_runAheadLowerCount; + if (m_runAheadLowerCount >= decreaseAfter) { + m_runAheadLowerCount = 0; + return m_runAheadLowerPeak; } + return oldRunAhead; } + m_runAheadLowerCount = 0; + return askedRunAhead; +} - - if (latencyLogicModel == 0) - { - return maxLatency; +void ConnectionManager::noteRunAheadApplied(Int oldRunAhead, Int newRunAhead) { + if (oldRunAhead == newRunAhead) { + return; } - else if (latencyLogicModel == 1) - { - NetworkMesh* pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + m_runAheadStillInitial = FALSE; +} +#endif - if (pMesh != nullptr) - { - Real maxGOLatency = (pMesh->getMaximumLatency() / 1000.f); - return maxLatency > maxGOLatency ? maxLatency : maxGOLatency; - } - else - { - return maxLatency; - } - } - else if (latencyLogicModel == 2) - { - NetworkMesh* pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); - if (pMesh != nullptr) - { - Real maxGOLatency = (pMesh->getMaximumHistoricalLatency() / 1000.f); - return maxLatency > maxGOLatency ? maxLatency : maxGOLatency; - } - else - { - return maxLatency; - } - } - else if (latencyLogicModel == 3) +// The average of the two highest in-band round trips, our own slot carries the one to our slowest peer. +Real ConnectionManager::getMaximumLatency() +{ + Real lat1 = 0.0f; + Real lat2 = 0.0f; + for (Int i = 0; i < MAX_SLOTS; ++i) { - NetworkMesh* pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); - - if (pMesh != nullptr) - { - Real maxGOLatency = (pMesh->getMaximumLatency() / 1000.f); - return maxGOLatency; - } - else + if (!isPlayerConnected(i)) { - return maxLatency; + continue; } - } - else if (latencyLogicModel == 4) - { - NetworkMesh* pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); - if (pMesh != nullptr) + if (m_latencyAverages[i] > lat1) { - Real maxGOLatency = (pMesh->getMaximumHistoricalLatency() / 1000.f); - return maxGOLatency; + lat2 = lat1; + lat1 = m_latencyAverages[i]; } - else + else if (m_latencyAverages[i] > lat2) { - return maxLatency; + lat2 = m_latencyAverages[i]; } } - - return maxLatency; + return (lat1 + lat2) / 2.0f; } void ConnectionManager::getMinimumFps(Int &minFps, Int &minFpsPlayer) { diff --git a/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp b/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp index 16e45c9e367..68b2c5941a7 100644 --- a/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp +++ b/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp @@ -26,6 +26,7 @@ #include "PreRTS.h" // This must go first in EVERY cpp file in the GameEngine +#include #include #include "GameNetwork/FrameMetrics.h" @@ -183,7 +184,9 @@ void FrameMetrics::doPerFrameMetrics(UnsignedInt frame) { // beyond size limit? remove the oldest //NetworkLog(ELogVerbosity::LOG_DEBUG, "SIZE of pending latencies: %ld and %ld", m_mapPendingLatenciesSorted.size(), m_mapPendingLatenciesLookup.size()); - if (m_mapPendingLatenciesSorted.size() > TheGlobalData->m_networkLatencyHistoryLength) + // the table holds the frame infos in the air, which is independent of how many samples are averaged + const UnsignedInt pendingCapacity = (std::max)(TheGlobalData->m_networkLatencyHistoryLength, (UnsignedInt)(MAX_FRAMES_AHEAD * 2)); + if (m_mapPendingLatenciesSorted.size() > pendingCapacity) { // remove oldest frame UnsignedInt oldestFrame = m_mapPendingLatenciesSorted.begin()->first; @@ -221,7 +224,7 @@ void FrameMetrics::processLatencyResponse(UnsignedInt frame) // beyond size limit? remove the oldest //NetworkLog(ELogVerbosity::LOG_DEBUG, "SIZE of actual latencies: %ld and %ld/%ld", m_mapLatenciesLookup.size(), m_mapLatenciesSorted.size(), TheGlobalData->m_networkLatencyHistoryLength); - if (m_mapLatenciesLookup.size() > TheGlobalData->m_networkLatencyHistoryLength) + while (!m_mapLatenciesSorted.empty() && m_mapLatenciesLookup.size() > TheGlobalData->m_networkLatencyHistoryLength) // more than one when the window shrank { // remove oldest frame UnsignedInt oldestFrame = m_mapLatenciesSorted.begin()->first; @@ -278,7 +281,25 @@ void FrameMetrics::addCushion(Int cushion) { } Int FrameMetrics::getAverageFPS() { +#if defined(GENERALS_ONLINE) + enum { MAX_MEDIAN_SAMPLES = 128 }; + Real sorted[MAX_MEDIAN_SAMPLES]; + UnsignedInt count = TheGlobalData->m_networkFPSHistoryLength; + if (count > MAX_MEDIAN_SAMPLES) { + count = MAX_MEDIAN_SAMPLES; + } + if (count == 0) { + return (Int)m_averageFps; + } + + for (UnsignedInt i = 0; i < count; ++i) { + sorted[i] = m_fpsList[i]; + } + std::nth_element(sorted, sorted + count / 2, sorted + count); + return (Int)sorted[count / 2]; +#else return (Int)m_averageFps; +#endif } Real FrameMetrics::getAverageLatency() { diff --git a/Core/GameEngine/Source/GameNetwork/GameSpy/LobbyUtils.cpp b/Core/GameEngine/Source/GameNetwork/GameSpy/LobbyUtils.cpp index 93892fe49f4..82f233ed591 100644 --- a/Core/GameEngine/Source/GameNetwork/GameSpy/LobbyUtils.cpp +++ b/Core/GameEngine/Source/GameNetwork/GameSpy/LobbyUtils.cpp @@ -381,7 +381,7 @@ static void gameTooltip(GameWindow* window, tooltip.concat(region); UnicodeString latency; - latency.format(L"\nLatency: %d (%d frames)\n", lobbyEntry.latency, ConvertMSLatencyToGenToolFrames(lobbyEntry.latency)); + latency.format(L"\nLatency: %d [%dms]\n", ConvertMSLatencyToFrames(lobbyEntry.latency), lobbyEntry.latency); tooltip.concat(latency); #else tooltip.format(TheGameText->fetch("TOOLTIP:GameInfoGameName"), room->getGameName().str()); diff --git a/Core/GameEngine/Source/GameNetwork/Network.cpp b/Core/GameEngine/Source/GameNetwork/Network.cpp index 9f321d53b54..2b22d47510b 100644 --- a/Core/GameEngine/Source/GameNetwork/Network.cpp +++ b/Core/GameEngine/Source/GameNetwork/Network.cpp @@ -126,6 +126,7 @@ class Network : public NetworkInterface Bool isFrameDataReady(void); #if defined(GENERALS_ONLINE) virtual Bool isGameExitPending() const override { return m_localStatus == NETLOCALSTATUS_POSTGAME; } + virtual UnsignedInt getMeasuredLogicRate() override; #endif virtual Bool isStalling(); void parseUserList(const GameInfo* game); @@ -228,6 +229,11 @@ class Network : public NetworkInterface Int m_lastExecutionFrame; ///< The highest frame number that a command could have been executed on. Int m_lastFrameCompleted; Bool m_didSelfSlug; +#if defined(GENERALS_ONLINE) + DWORD m_rateSampleMs; + UnsignedInt m_rateSampleFrame; + UnsignedInt m_measuredLogicRate; +#endif __int64 m_perfCountFreq; ///< The frequency of the performance counter. __int64 m_nextFrameTime; ///< When did we execute the last frame? For slugging the GameLogic... @@ -382,6 +388,9 @@ void Network::init() #if defined(GENERALS_ONLINE) m_matchKeysRequired = FALSE; m_matchStartFailed = FALSE; + m_rateSampleMs = 0; + m_rateSampleFrame = 0; + m_measuredLogicRate = 0; #endif QueryPerformanceFrequency((LARGE_INTEGER*)&m_perfCountFreq); @@ -732,7 +741,13 @@ void Network::processFrameSynchronizedNetCommand(NetCommandRef* msg) { } void Network::processRunAheadCommand(NetRunAheadCommandMsg* msg) { +#if defined(GENERALS_ONLINE) + const Int previousRunAhead = m_runAhead; +#endif m_runAhead = msg->getRunAhead(); +#if defined(GENERALS_ONLINE) + m_conMgr->noteRunAheadApplied(previousRunAhead, m_runAhead); +#endif m_frameRate = msg->getFrameRate(); time_t frameGrouping = (1000 * m_runAhead) / m_frameRate; // number of miliseconds between packet sends frameGrouping = frameGrouping / 2; // since we only want the latency for one way to be a factor. @@ -923,7 +938,12 @@ Bool Network::timeForNewFrame() { */ if (m_conMgr != NULL) { Real cushion = m_conMgr->getMinimumCushion(); +#if defined(GENERALS_ONLINE) + // the controller keeps a margin of one frame, slow down when the commands of the slowest player arrive less than that early + Real runAheadPercentage = 1.0f; +#else Real runAheadPercentage = m_runAhead * (TheGlobalData->m_networkRunAheadSlack / (Real)100.0); // If we are at least 50% into our slack, we need to slow down. +#endif if (cushion < runAheadPercentage) { __int64 oldFrameDelay = frameDelay; frameDelay += oldFrameDelay / 10; // temporarily decrease the frame rate by 20%. @@ -938,9 +958,13 @@ Bool Network::timeForNewFrame() { if (curTime >= m_nextFrameTime) { NetworkLog(ELogVerbosity::LOG_DEBUG, "Allowing a new frame, frameDelay = %I64d, curTime - m_nextFrameTime = %I64d", frameDelay, curTime - m_nextFrameTime); -// if (m_nextFrameTime + frameDelay < curTime) { - if (m_nextFrameTime + frameDelay < curTime) { - //if ((m_nextFrameTime + ((GENERALS_ONLINE_HIGH_FPS_FRAME_MULTIPLIER * 2) * frameDelay)) < curTime) { + __int64 catchUpFrames = 1; +#if defined(GENERALS_ONLINE) + if (m_conMgr != nullptr) { + catchUpFrames = m_conMgr->getNetworkSettings().input_delay.catch_up_frames; + } +#endif + if (m_nextFrameTime + catchUpFrames * frameDelay < curTime) { // If we get too far behind on our framerate we need to reset the nextFrameTime thing. m_nextFrameTime = curTime; NetworkLog(ELogVerbosity::LOG_DEBUG, "Initializing m_nextFrameTime to %I64d", m_nextFrameTime); @@ -1216,6 +1240,22 @@ Int Network::getSlotAverageFPS(Int slot) { return -1; } +#if defined(GENERALS_ONLINE) +UnsignedInt Network::getMeasuredLogicRate() { + const DWORD now = timeGetTime(); + const UnsignedInt frame = TheGameLogic->getFrame(); + if (m_rateSampleMs == 0 || frame < m_rateSampleFrame) { + m_rateSampleMs = now; + m_rateSampleFrame = frame; + } else if (now - m_rateSampleMs >= 1000) { + m_measuredLogicRate = (UnsignedInt)(((frame - m_rateSampleFrame) * 1000.0f) / (Real)(now - m_rateSampleMs) + 0.5f); + m_rateSampleMs = now; + m_rateSampleFrame = frame; + } + return m_measuredLogicRate; +} +#endif + const ConnectionStats* Network::getConnectionStats(Int slot) const { if (m_conMgr != nullptr) { return m_conMgr->getConnectionStats(slot); diff --git a/GeneralsMD/Code/GameEngine/Include/GameClient/InGameUI.h b/GeneralsMD/Code/GameEngine/Include/GameClient/InGameUI.h index 872a4dab664..d5c966ceac8 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameClient/InGameUI.h +++ b/GeneralsMD/Code/GameEngine/Include/GameClient/InGameUI.h @@ -834,6 +834,7 @@ class InGameUI : public SubsystemInterface, public Snapshot Color m_networkLatencyColor; Color m_networkLatencyDropColor; UnsignedInt m_lastNetworkLatencyFrames; + UnsignedInt m_lastNetworkLogicRate; // Render FPS Counter DisplayString* m_renderFpsString; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h index e439210828b..57da3030398 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h @@ -36,7 +36,6 @@ std::wstring WidenFormatSpecifiers(const std::wstring& format); int RoundUpLatencyToFrameInterval(int latency, int frameInterval); int ConvertMSLatencyToFrames(int ms); -int ConvertMSLatencyToGenToolFrames(int ms); // NGMP_NOTE: Plr Templates look like this: /* | NAME | GAME INDEX | SERVICE INDEX | PLAYABLE | diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h index 0b69979da0a..d370e440bc5 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h @@ -55,8 +55,6 @@ class PlayerConnection void SendACPacket(const void* pData, uint32_t dataLen); - void UpdateLatencyHistogram(); - void TickProbe(int64_t nowMs); void HandleProbe(const uint8_t* payload, size_t len, int64_t arrivalMs); void ResetProbe(); @@ -81,21 +79,17 @@ class PlayerConnection int Recv(SteamNetworkingMessage_t** pMsg, int maxMessages = 255); - int GetHighestHistoricalLatency() + // the p75 probe round trip of the stats window, -1 with too few samples: a spike or a lossy spell must not set the start run ahead + int GetSeedLatency(int minSamples = k_seedMinSamples) const { - int highestLatency = 0; - for (int latencyHistory : m_vecLatencyHistory) + if (m_probeStats.getSampleCount() < minSamples) { - if (latencyHistory > highestLatency) - { - highestLatency = latencyHistory; - } + return -1; } - - return highestLatency; + return static_cast(m_probeStats.getLatencyPercentile(k_seedPercentile) + 0.5f); } - - std::vector m_vecLatencyHistory; + static constexpr int k_seedMinSamples = 10; // the probes a lobby round trip, a jitter or a rating needs, 2.5 s at the lobby interval + static constexpr int k_seedPercentile = 75; float GetSustainedLossPercent(int windowMs); ///< The worse direction of the packet feedback loss from at least 3 reports, -1 if unknown. std::string GetStats(); @@ -221,22 +215,6 @@ class NetworkMesh m_cbOnConnected = nullptr; } - int getMaximumLatency() - { - int highestLatency = 0; - - for (auto& kvPair : m_mapConnections) - { - PlayerConnection& conn = kvPair.second; - if (conn.GetLatency() > highestLatency) - { - highestLatency = conn.GetLatency(); - } - } - - return highestLatency; - } - // The signing keys, see MatchKeys.h bool BeginLobbyKeys(); ///< Makes this client's key, call when it joins or creates a lobby. void TickLobbyKeys(int64_t nowMs); ///< Announces the key to the peers that lack it and tells them which keys we hold. @@ -254,20 +232,21 @@ class NetworkMesh const MatchKeys& GetMatchKeys() const { return m_matchKeys; } MatchKeys& GetMatchKeys() { return m_matchKeys; } - Real getMaximumHistoricalLatency() + // the highest p75 probe round trip of any connection over the probe stats window, 0 if none has enough samples + int getSeedProbeLatency() { int highestLatency = 0; for (auto& kvPair : m_mapConnections) { - PlayerConnection& conn = kvPair.second; - if (conn.GetHighestHistoricalLatency() > highestLatency) + const int seed = kvPair.second.GetSeedLatency(); + if (seed > highestLatency) { - highestLatency = conn.GetHighestHistoricalLatency(); + highestLatency = seed; } } - return Real(highestLatency); + return highestLatency; } diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index 3a0681e309d..60bb3e255d0 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -324,7 +324,7 @@ struct RegionResponse struct ServiceConfig { - // Peer connections: signalling, NAT traversal, latency probes, and the per-connection ping history + // Peer connections: signalling, NAT traversal and latency probes struct Connection { bool relay_only = false; @@ -334,18 +334,8 @@ struct ServiceConfig bool anticheat_transport = false; // game traffic through the anticheat plugin's transport int probe_lobby_interval_ms = 250; // time between latency probes sent to each peer while not in a match int probe_match_interval_ms = 1000; // the same in a match, the command acks already measure the round trip - int latency_history_ms = 20000; } connection; - // Not read from the service: the live values - int network_latency_logic_model = 2; - bool ibra_ra_tweaks = false; - float ibra_minslack_default = 0.25f; - float ibra_maxslack_default = 1.f; - float ibra_minslack_greaterthan300ms = 0.35f; - float ibra_maxslack_greaterthan300ms = 1.f; - float ibra_minslack_greaterthan200ms = 0.3f; - float ibra_maxslack_greaterthan200ms = 1.f; using Send = NetworkSettings::Send; using InputDelay = NetworkSettings::InputDelay; using Disconnect = NetworkSettings::Disconnect; @@ -429,7 +419,6 @@ inline void ReadServiceConfig(const nlohmann::json& j, ServiceConfig& c) ReadServiceConfigValue(s, "anticheat_transport", c.connection.anticheat_transport); ReadServiceConfigValue(s, "probe_lobby_interval_ms", c.connection.probe_lobby_interval_ms); ReadServiceConfigValue(s, "probe_match_interval_ms", c.connection.probe_match_interval_ms); - ReadServiceConfigValue(s, "latency_history_ms", c.connection.latency_history_ms); } if (j.contains("send")) { @@ -450,11 +439,14 @@ inline void ReadServiceConfig(const nlohmann::json& j, ServiceConfig& c) ReadServiceConfigValue(s, "min_frames", c.input_delay.min_frames); ReadServiceConfigValue(s, "decision_interval_ms", c.input_delay.decision_interval_ms); ReadServiceConfigValue(s, "fps_history_samples", c.input_delay.fps_history_samples); + ReadServiceConfigValue(s, "catch_up_frames", c.input_delay.catch_up_frames); ReadServiceConfigValue(s, "slack_percent", c.input_delay.slack_percent); if (s.contains("formula")) { const nlohmann::json& f = s["formula"]; - ReadServiceConfigValue(f, "latency_history_samples", c.input_delay.formula.latency_history_samples); + ReadServiceConfigValue(f, "latency_window_ms", c.input_delay.formula.latency_window_ms); + ReadServiceConfigValue(f, "decrease_after_decisions", c.input_delay.formula.decrease_after_decisions); + ReadServiceConfigValue(f, "repair_budget", c.input_delay.formula.repair_budget); } } if (j.contains("disconnect")) @@ -482,18 +474,22 @@ inline void ReadServiceConfig(const nlohmann::json& j, ServiceConfig& c) // Keeps values from the service inside the ranges the game can run with inline void ClampServiceConfig(ServiceConfig& c) { - c.connection.latency_history_ms = (std::max)(c.connection.latency_history_ms, 1000); c.connection.probe_lobby_interval_ms = std::clamp(c.connection.probe_lobby_interval_ms, 50, 5000); c.connection.probe_match_interval_ms = std::clamp(c.connection.probe_match_interval_ms, 50, 10000); c.send.redundancy_window_ms = std::clamp(c.send.redundancy_window_ms, 0, 5000); c.send.redundancy_burst_ms = std::clamp(c.send.redundancy_burst_ms, 0, 1000); c.send.frames_per_packet_lossy = std::clamp(c.send.frames_per_packet_lossy, 0, 8); - c.input_delay.min_frames = std::clamp(c.input_delay.min_frames, 1, 64); + c.input_delay.formula.decrease_after_decisions = std::clamp(c.input_delay.formula.decrease_after_decisions, 0, 20); + c.input_delay.catch_up_frames = std::clamp(c.input_delay.catch_up_frames, 1, 4); + c.input_delay.min_frames = std::clamp(c.input_delay.min_frames, 0, 30); // the start run ahead is at least 30, a higher floor would make configs disagree c.input_delay.slack_percent = std::clamp(c.input_delay.slack_percent, 0, 200); c.input_delay.decision_interval_ms = std::clamp(c.input_delay.decision_interval_ms, 100, 60000); c.input_delay.fps_history_samples = std::clamp(c.input_delay.fps_history_samples, 1, 128); - c.input_delay.formula.latency_history_samples = std::clamp(c.input_delay.formula.latency_history_samples, 128, 3600); // FrameMetrics drops pending acks when the window is smaller than the frames in flight + if (c.input_delay.formula.latency_window_ms > 0) + { + c.input_delay.formula.latency_window_ms = std::clamp(c.input_delay.formula.latency_window_ms, 30 * 1000 / GENERALS_ONLINE_HIGH_FPS_LIMIT + 1, 60000); + } c.send.frames_per_packet = std::clamp(c.send.frames_per_packet, 1, 16); c.disconnect.freeze_before_screen_ms = std::clamp(c.disconnect.freeze_before_screen_ms, 1000, 30000); diff --git a/GeneralsMD/Code/GameEngine/Source/Common/GlobalData.cpp b/GeneralsMD/Code/GameEngine/Source/Common/GlobalData.cpp index 04063cf38f6..1305b0e9197 100644 --- a/GeneralsMD/Code/GameEngine/Source/Common/GlobalData.cpp +++ b/GeneralsMD/Code/GameEngine/Source/Common/GlobalData.cpp @@ -1031,7 +1031,11 @@ GlobalData::GlobalData() m_networkFPSHistoryLength = 30; m_networkLatencyHistoryLength = 200; m_networkRunAheadMetricsTime = 500; +#if defined(GENERALS_ONLINE) + m_networkCushionHistoryLength = 20; // 333 ms at 60 Hz, what retail's 10 frames at 30 Hz covered +#else m_networkCushionHistoryLength = 10; +#endif m_networkRunAheadSlack = 10; m_networkKeepAliveDelay = 20; m_networkDisconnectTime = 5000; diff --git a/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp b/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp index ea6a22d989b..65ab0d87426 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp @@ -1179,6 +1179,7 @@ InGameUI::InGameUI() m_networkLatencyColor = GameMakeColor(173, 216, 255, 255); m_networkLatencyDropColor = GameMakeColor(0, 0, 0, 255); m_lastNetworkLatencyFrames = ~0u; + m_lastNetworkLogicRate = ~0u; m_renderFpsString = nullptr; m_renderFpsLimitString = nullptr; @@ -6893,7 +6894,8 @@ void InGameUI::refreshNetworkLatencyResources() { m_networkLatencyString = TheDisplayStringManager->newDisplayString(); m_lastNetworkLatencyFrames = ~0u; - } + m_lastNetworkLogicRate = ~0u; + } m_networkLatencyPointSize = TheGlobalData->m_networkLatencyFontSize; Int adjustedNetworkLatencyFontSize = TheGlobalLanguageData->adjustFontSize(m_networkLatencyPointSize); @@ -7037,26 +7039,16 @@ void InGameUI::updateRenderFpsString() void InGameUI::drawNetworkLatency(Int& x, Int& y) { #if defined(GENERALS_ONLINE) - const UnsignedInt actualLatencyInMS = TheNetwork->getRunAhead() * (1000 / GENERALS_ONLINE_HIGH_FPS_LIMIT); - const UnsignedInt actualFrames = ConvertMSLatencyToFrames(actualLatencyInMS); - const UnsignedInt gentoolFrames = ConvertMSLatencyToGenToolFrames(actualLatencyInMS); - - //bool bIsSelfSlugged = TheNetwork->IsSlugging(); + const UnsignedInt networkLatencyFrames = TheNetwork->getRunAhead(); + const UnsignedInt logicRate = TheNetwork->getMeasuredLogicRate(); - if (gentoolFrames != m_lastNetworkLatencyFrames) + if (networkLatencyFrames != m_lastNetworkLatencyFrames || logicRate != m_lastNetworkLogicRate) { UnicodeString latencyStr; - - if (actualFrames != gentoolFrames) - { - latencyStr.format(L"[%u] - [%ums - %u]", TheNetwork->getFrameRate(), actualLatencyInMS, actualFrames); - } - else - { - latencyStr.format(L"%u [%ums][L: %u]", gentoolFrames, actualLatencyInMS, TheNetwork->getFrameRate()); - } + latencyStr.format(L"%u [%ums] [L%u]", networkLatencyFrames, (networkLatencyFrames * 1000 + GENERALS_ONLINE_HIGH_FPS_LIMIT / 2) / GENERALS_ONLINE_HIGH_FPS_LIMIT, logicRate); m_networkLatencyString->setText(latencyStr); - m_lastNetworkLatencyFrames = gentoolFrames; + m_lastNetworkLatencyFrames = networkLatencyFrames; + m_lastNetworkLogicRate = logicRate; } #else const UnsignedInt networkLatencyFrames = TheNetwork->getRunAhead(); @@ -7070,8 +7062,6 @@ void InGameUI::drawNetworkLatency(Int& x, Int& y) } #endif - - // TheSuperHackers @info at the HUD anchor this draws inline and advances x otherwise uses configured position if (isAtHudAnchorPos(m_networkLatencyPosition)) { @@ -7272,7 +7262,7 @@ namespace struct OverlayCache { - OverlayCache() : valid(FALSE), mode(0), refreshedMs(0), lineH(12), rowCount(0) + OverlayCache() : valid(FALSE), mode(0), refreshedMs(0), lineH(12), logicRate(0), rowCount(0) { for (Int i = 0; i < k_strTotal; ++i) { @@ -7288,6 +7278,7 @@ namespace Int mode; UnsignedInt refreshedMs; Int lineH; + UnsignedInt logicRate; Int rowCount; NetworkOverlayInfo header; UnicodeString waitingOn; @@ -7406,6 +7397,7 @@ void InGameUI::drawNetworkOverlay(Int mode) overlaySetText(pMeasure, UnicodeString(L"Wg")); pMeasure->getSize(&lineW, &lineH); cache.lineH = (std::max)(lineH, 12); + cache.logicRate = TheNetwork->getMeasuredLogicRate(); TheNetwork->getOverlayInfo(-1, cache.header); for (Int slot = 0; slot < MAX_SLOTS; ++slot) @@ -7568,7 +7560,7 @@ void InGameUI::drawNetworkOverlay(Int mode) { Int x = left; UnicodeString text; - text.format(L"Input delay %d ms", delayMs); + text.format(L"Input delay %d ms, logic rate %u Hz", delayMs, cache.logicRate); x += overlayDrawText(k_strHeaderBase + 0, x, y, text, header.stalledNow ? overlayColor(ConnectionRating::SEVERITY_BAD) : neutral, dropColor) + segmentGap; if (header.stalledNow && !waitingOn.isEmpty()) diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp index bf00a650e6c..62826ec555e 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp @@ -466,7 +466,8 @@ void NGMPGame::launchGame(void) PopBackToLobby(); return; } - TheNetwork->SeedLatencyData(pMesh->getMaximumHistoricalLatency()); + // One spike or lossy spell must not start the match at a huge run ahead. + TheNetwork->SeedLatencyData(pMesh->getSeedProbeLatency()); // TODO_NGMP: Do we really care about these values anymore TheNetwork->setLocalAddress(getLocalIP(), 8888); diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMP_Helpers.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMP_Helpers.cpp index 77223c06347..c6014600128 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMP_Helpers.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMP_Helpers.cpp @@ -335,11 +335,6 @@ int ConvertMSLatencyToFrames(int ms) return (int)ceil((ms / 1000.f) * (float)GENERALS_ONLINE_HIGH_FPS_LIMIT); } -int ConvertMSLatencyToGenToolFrames(int ms) -{ - return (int)ceil((float)ConvertMSLatencyToFrames(ms) / (float)GENERALS_ONLINE_HIGH_FPS_FRAME_MULTIPLIER); -} - #include #include #include diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp index c4341e7d817..477e0d7b5e8 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp @@ -1505,12 +1505,11 @@ void NetworkMesh::Tick() const int64_t nowMs = NowSteadyMs(); - // update connection histograms + // probe every connection for (auto& kvPair : m_mapConnections) { PlayerConnection& conn = kvPair.second; conn.TickProbe(nowMs); - conn.UpdateLatencyHistogram(); } TickLobbyKeys(nowMs); @@ -1907,32 +1906,6 @@ void PlayerConnection::SendACPacket(const void* pData, uint32_t dataLen) } } -void PlayerConnection::UpdateLatencyHistogram() -{ - int histogram_duration = 20000; - - if (NGMP_OnlineServicesManager::GetInstance() != nullptr) - { - ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - histogram_duration = serviceConf.connection.latency_history_ms; - } - - - // update latency history, from the mesh probe once it has samples - int currLatency = (m_probeStats.getSampleCount() > 0) ? static_cast(m_probeStats.getLatency()) : GetLatency(); -#if defined(GENERALS_ONLINE_HIGH_FPS_SERVER) - const int connectionHistoryLength = histogram_duration / 16; // ~20 sec worth of frames at 60fps (default) -#else - const int connectionHistoryLength = histogram_duration / 33; // ~20 sec worth of frames at 30fps (default) -#endif - - if (m_vecLatencyHistory.size() >= connectionHistoryLength) - { - m_vecLatencyHistory.erase(m_vecLatencyHistory.begin()); - } - m_vecLatencyHistory.push_back(currLatency); -} - void PlayerConnection::Close() { if (m_ConnectionType == EConnectionType::BuiltIn_ValveSockets) @@ -2224,30 +2197,13 @@ float PlayerConnection::GetSustainedLossPercent(int windowMs) int PlayerConnection::GetJitter() { - int sumDelta = 0; - int count = 0; - int prev = -1; - for (int sample : m_vecLatencyHistory) + // smoothed round trip variation of the probe over its stats window + if (m_probeStats.getSampleCount() < 10) { - if (sample >= 0) - { - if (prev >= 0) - { - sumDelta += std::abs(sample - prev); - ++count; - } - prev = sample; - } - else - { - prev = -1; // gap in valid data - } - } - - if (count < 10) return -1; + } - return sumDelta / count; + return static_cast(m_probeStats.getJitter()); } int PlayerConnection::ComputeConnectionRating() From b96341974985165adbcecfb7ac68eca99d3c2570 Mon Sep 17 00:00:00 2001 From: Jacob Ledbetter Date: Sat, 15 Aug 2026 13:44:02 -0600 Subject: [PATCH 11/12] bugfix(network): Sort fast path in NetCommandList by sort number (#3052) --- Core/GameEngine/Source/GameNetwork/NetCommandList.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Core/GameEngine/Source/GameNetwork/NetCommandList.cpp b/Core/GameEngine/Source/GameNetwork/NetCommandList.cpp index 1ca2d4a87e2..f2a95b7e4d0 100644 --- a/Core/GameEngine/Source/GameNetwork/NetCommandList.cpp +++ b/Core/GameEngine/Source/GameNetwork/NetCommandList.cpp @@ -167,12 +167,15 @@ NetCommandRef * NetCommandList::addMessage(NetCommandRef *&msg) { // So saving the placement of the last message inserted can give us a huge boost in // efficiency. NetCommandRef *theNext = m_lastMessageInserted->getNext(); + + // TheSuperHackers @bugfix CryoTheRenegade 03/08/2026 Keep both cached + // insertion boundaries consistent with the full scan's polymorphic sort key. if ((m_lastMessageInserted->getCommand()->getNetCommandType() == msg->getCommand()->getNetCommandType()) && (m_lastMessageInserted->getCommand()->getPlayerID() == msg->getCommand()->getPlayerID()) && - isCommandIdNewer(msg->getCommand()->getID(), m_lastMessageInserted->getCommand()->getID()) && + isCommandIdNewer(msg->getCommand()->getSortNumber(), m_lastMessageInserted->getCommand()->getSortNumber()) && ((theNext == nullptr) || ((theNext->getCommand()->getNetCommandType() > msg->getCommand()->getNetCommandType()) || (theNext->getCommand()->getPlayerID() > msg->getCommand()->getPlayerID()) || - isCommandIdNewer(theNext->getCommand()->getID(), msg->getCommand()->getID())))) { + isCommandIdNewer(theNext->getCommand()->getSortNumber(), msg->getCommand()->getSortNumber())))) { // Make sure this command isn't already in the list. if (isEqualCommandMsg(m_lastMessageInserted->getCommand(), msg->getCommand())) { From 3eb6426f5b4f4a6a4e3adf51f989525604e023cc Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Thu, 22 Oct 2026 07:03:30 +0200 Subject: [PATCH 12/12] feat(network): Control arrival cushion and starting runahead --- .../Include/GameNetwork/ConnectionManager.h | 63 +- .../Include/GameNetwork/ConnectionStats.h | 105 ++- .../Include/GameNetwork/NetCommandMsg.h | 19 + .../Include/GameNetwork/NetPacketStructs.h | 5 + .../Include/GameNetwork/NetworkDefs.h | 5 + .../Include/GameNetwork/NetworkInterface.h | 2 + .../Include/GameNetwork/NetworkSettings.h | 4 + .../Source/GameNetwork/ConnectionManager.cpp | 657 +++++++++++++++- .../Source/GameNetwork/ConnectionStats.cpp | 372 +++++++++ .../Source/GameNetwork/FrameMetrics.cpp | 12 +- .../Source/GameNetwork/NetCommandMsg.cpp | 8 + .../Source/GameNetwork/NetPacketStructs.cpp | 23 + .../GameEngine/Source/GameNetwork/Network.cpp | 139 +++- .../GameEngine/Include/GameLogic/GameLogic.h | 1 + .../GeneralsOnline/ConnectionRating.h | 5 +- .../GameNetwork/GeneralsOnline/NGMP_include.h | 3 +- .../GameNetwork/GeneralsOnline/NetworkMesh.h | 66 ++ .../GeneralsOnline/OnlineServices_Init.h | 6 + .../GUICallbacks/Menus/WOLGameSetupMenu.cpp | 7 +- .../GameEngine/Source/GameClient/InGameUI.cpp | 76 +- .../Source/GameLogic/System/GameLogic.cpp | 108 ++- .../GeneralsOnline/ConnectionRating.cpp | 15 +- .../GameNetwork/GeneralsOnline/NGMPGame.cpp | 6 + .../GeneralsOnline/NetworkMesh.cpp | 710 +++++++++++++++++- .../GeneralsOnline/NextGenTransport.cpp | 10 + .../GeneralsOnline/OnlineServices_Init.cpp | 16 + 26 files changed, 2352 insertions(+), 91 deletions(-) diff --git a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h index 667ff4f445e..91e1a3c037b 100644 --- a/Core/GameEngine/Include/GameNetwork/ConnectionManager.h +++ b/Core/GameEngine/Include/GameNetwork/ConnectionManager.h @@ -161,6 +161,9 @@ class ConnectionManager Int getSlotAverageFPS(Int slot); #if defined(GENERALS_ONLINE) void noteStalling(Bool stalling, UnsignedInt frame); ///< Called every network update with whether the game is waiting for commands. + void noteScreenshotCapture(Bool uploadFollows); ///< The main thread captures a screenshot, the stalls until its upload is done are not the network's. + void noteScreenshotDone(); ///< The upload of the screenshot ended, the stalls of the next 2 s are still not the network's. + Int getRemoteLookAhead(UnsignedInt frame, Int maxFrames, Bool &overdue); ///< Fewest frames after this one a remote player's commands are complete for, -1 without a remote player. #endif const ConnectionStats* getConnectionStats(Int slot) const; ///< Returns nullptr when there is no connection for the slot. @@ -203,6 +206,7 @@ class ConnectionManager // void doPerFrameMetrics(UnsignedInt frame); void getMinimumFps(Int &minFps, Int &minFpsPlayer); ///< Returns the smallest FPS in the m_fpsAverages list. Real getMaximumLatency(); ///< Returns the average of the two highest average latencies between players. + Real getLiveProbeLatency(); ///< Seconds, the highest probe round trip to a player still in the game, 0 if none is known. void requestFrameDataResend(Int playerID, UnsignedInt frame); ///< request of this player that he send the specified frame's data. UnsignedShort generateImpersonatedCommandID(Int playerID, UnsignedInt frame); @@ -250,6 +254,7 @@ class ConnectionManager Int m_minFpsPlayer; Int m_minFps; UnsignedInt m_smallestPacketArrivalCushion; + time_t m_lastRunAheadDecisionTime; ///< The last run ahead decision of this match, 0 before the first. Bool m_didSelfSlug; #if defined(GENERALS_ONLINE) @@ -271,7 +276,6 @@ class ConnectionManager Bool isPeerActive(Int slot) const; void onDirectFrameAck(UnsignedInt frame, Int peerSlot); std::map m_frameAckedMasks; ///< Frame -> the peers that acked its frame info so far. - Bool isDirectSendActive(); Bool m_directSendActive; time_t m_directSendCheckedAt; UnsignedInt m_formerRouterMask; ///< Slots that were the packet router in this match, their last packets can still be on the way. @@ -353,19 +357,74 @@ class ConnectionManager Bool m_stallActive; time_t m_stallStartTime; UnsignedInt m_stallMissingMask; + Bool m_stallWaitedOnRemote; ///< The current stall had a remote player's commands missing Int m_lastKnownRunAhead; + + // cushion controller (input_delay.controller), every player measures per sender, the packet router sums the cushions of each pair + Int m_ctlWindowMin[MAX_SLOTS]; ///< Lowest cushion per sender of the commands issued under the current run ahead since the last report, NO_CUSHION if none. + UnsignedInt m_ctlValidFromExec[MAX_SLOTS]; ///< Commands of a sender for an earlier execution frame were issued under the previous run ahead. + Int m_ctlLateCount[MAX_SLOTS]; ///< Late commands (cushion <= 0) per sender since the last report. + Int m_ctlStallMs; ///< Ms of the waits of at least a frame for a remote player's commands since the last report. + time_t m_ctlStallMark; ///< Stall time before this was reported already. + time_t m_ctlScreenshotUntil; ///< Reports are flagged as holding a local screenshot capture until one is taken at or after this, 0 if none. + time_t m_screenshotStallUntil; ///< A stall that starts before this follows a local screenshot capture + Bool m_ctlScreenshotSeen; ///< Router: a report since the last decision was flagged as holding a screenshot capture. + Bool m_ctlSeeded; + Int m_ctlSeedWait; + Int m_ctlMin[MAX_SLOTS][MAX_SLOTS]; ///< Router: lowest cushion of the reports since the last decision, [reporting player][sender]. + Int m_ctlReportLate[MAX_SLOTS]; ///< Router: late commands per reporting player since the last decision. + Int m_ctlReportStall[MAX_SLOTS]; ///< Router: the same stall ms per reporting player since the last decision, the decision uses the largest of them. + time_t m_ctlIgnoreUntil; ///< Router: reports arriving before this were measured under the previous run ahead, D after a change. + time_t m_ctlChangeTime; ///< Router: when the last run ahead change was decided. + time_t m_ctlReportTime[MAX_SLOTS]; ///< Router: when the last report counted since the change arrived per player, 0 = none yet. + time_t m_lastHeardMs[MAX_SLOTS]; ///< When the last packet of a player arrived, 0 = none yet. + time_t m_paceLastFreshMs[MAX_SLOTS]; ///< When the last new frame info of a player arrived, 0 = none yet. + Real m_paceGapAvgMs[MAX_SLOTS]; ///< Smoothed gap between the arrivals of new frame infos per player. + Real m_paceGapDevMs[MAX_SLOTS]; ///< Smoothed deviation of that gap. + time_t m_ctlOutageSince; ///< Router: since when a connected player is silent, 0 = no outage. + Int m_ctlSlowSlot; ///< Router: the slow producer whose stalls the last decision left out, -1 = none. + void updateControllerOutage(time_t now); ///< Router: starts or ends an outage, the controller holds through it. + time_t m_ctlLastPairTime; ///< Router: when a decision last saw a complete pair, the latency formula takes over without one. + time_t m_ctlLastWindowTime; ///< Router: when the last report window was taken in, its length is the time since. + RunAheadSpikeController m_spikeController; ///< Router: the baseline quantile, spike episodes and the cost rule. Bool m_runAheadStillInitial; ///< No run ahead change has taken effect yet, the first decisions apply without the decrease hold. Int m_pendingRunAhead; ///< Router: the run ahead of the last command it issued. UnsignedInt m_pendingExecFrame; ///< Router: the execution frame of that command. + void collectControllerReport(Int *lowest, Int *late, Int &stallMs, Bool &screenshot); ///< The lowest cushion per sender, the late commands and the stall ms since the last report. + void resetControllerWindow(Bool sinceChange); ///< Drops the reports of the window, and with sinceChange which players reported since the change. + Int decideControllerRunAhead(Int oldRunAhead, Int formulaRunAhead, Int floorRunAhead, time_t now, Int decreaseAfter); + Int discardControllerReport(); ///< Takes the own report of the window and drops it, returns its stall ms. + Bool m_fastDropStep; ///< Router: the last decision was a fast drop step, the next one waits the short dead time. Int applyDecreaseHold(Int oldRunAhead, Int askedRunAhead, Int decreaseAfter); public: void noteRunAheadApplied(Int oldRunAhead, Int newRunAhead); ///< The run ahead command took effect on this frame. + Int getRunAheadFloor() const; ///< The configured minimum, or the send interval frames + 3 + Int getRunAheadCap() const; ///< The run ahead cap in frames, input_delay.max_frames. + Bool acceptsRunAhead(Int runAhead) const; ///< An online match takes a run ahead between its floor and cap (0 only while alone). + Int computeStartRunAhead(Real roundTripMs, Bool oneWay) const; ///< The run ahead in frames estimateRunAheadMs asks for, between the floor and the cap. + Bool isDirectSendActive(Bool atStart = FALSE); ///< send.direct is on and every peer connection is up; atStart looks at the links right now and counts every peer in, before the match runs nobody is out + Bool isLinkUp(Int slot); ///< The mesh connection to the slot is up, a pair that fell back to the packet router is not. + Bool usesOneWayRunAhead(Bool atStart = FALSE); ///< The controller is on and direct sending is active right now: a command travels one way, not through the router. + Bool usesOneWayRunAheadForUser(Int64 userID); ///< The same for the link to one user: its connection is up and direct sending is on. + static Real estimateRunAheadMs(Real roundTripMs, const NetworkSettings::Send &sendSettings, const NetworkSettings::InputDelay &inputDelay, Bool oneWay); ///< The input delay a round trip needs, shared by the start run ahead, the controller seed and the connection rating. + void applyStartRunAhead(Int oldRunAhead, Int newRunAhead); ///< Frames 1 to the new run ahead - 1 have no commands, the frames after it wait for theirs. private: Bool m_shortGrouping; ///< The run ahead is within 2 frames of its floor, connections send at one frame - Int getRunAheadFloor() const; time_t getEffectiveDecisionIntervalMs(Int runAhead) const; ///< The longer of the configured floor and 2 x the run ahead in ms void updateShortGrouping(); + Int m_ctlLastReason; ///< Router: NetworkOverlayInfo::ControllerReason of the last controller decision. + Int m_ctlLastMinSum; ///< Router: what that decision saw, for the overlay. + Int m_ctlLastPairA; + Int m_ctlLastPairB; + Int m_ctlLastBase; + Int m_ctlLastCover; + Int m_ctlLastSpikesPerMin10; + Int m_ctlLastSpikeSizeMs; + Bool m_ctlLastInSpike; + Int m_changeDir; ///< Direction of the last run ahead change that took effect, 1 up, -1 down. + Int m_changeReason; ///< NetworkOverlayInfo::ChangeReason of that change. + time_t m_changeAppliedTime; ///< When it took effect on this machine, 0 = none yet. Int m_frameGroupingMs; public: Bool getOverlayInfo(Int slot, NetworkOverlayInfo &info); ///< Fills what the overlay and the connection rating need about one slot. diff --git a/Core/GameEngine/Include/GameNetwork/ConnectionStats.h b/Core/GameEngine/Include/GameNetwork/ConnectionStats.h index db57961c0d0..e613b18cbe9 100644 --- a/Core/GameEngine/Include/GameNetwork/ConnectionStats.h +++ b/Core/GameEngine/Include/GameNetwork/ConnectionStats.h @@ -168,13 +168,98 @@ class StallStats TimedRing m_events; }; +// Run ahead from the per window need: a forgetting quantile of normal windows is the baseline; spike episodes are bought only +// when they recur faster than one per lambda. Rules use window time, so the decision interval does not matter. +class RunAheadSpikeController +{ +public: + enum Reason { REASON_HOLD, REASON_BASELINE, REASON_COVER, REASON_UNCOVER, REASON_CONGESTION }; + + struct Decision + { + Int runAhead; + Int reason; ///< Reason. + Int base; ///< Frames, the q quantile of the normal windows. + Int cover; ///< Frames bought on top of the baseline for spikes. + Int spikesPerMin10; ///< Spike episodes per minute over the horizon, times 10. + Int spikeSizeMs; ///< The median spike height over the baseline in the horizon. + Bool inSpike; + Bool fastDrop; ///< The target is the recent need, the next decision does not wait for the long dead time. + }; + + RunAheadSpikeController(); + + void reset(); + void dropEpisode(); ///< An outage: the open episode is not the network's trend. + Decision onWindow(time_t now, Int windowMs, Int need, Bool stalled, Int stallMs, Real jitterMs, Int runAhead, Int marginFrames, Real frameMs, Int floorFrames, Int capFrames, Bool safeFastDrop, Bool boundRaise); + +private: + enum + { + BUCKETS = 128, // one frame each, the need is 0 to 127 frames + SAMPLE_RING = 16, // need samples of the open episode + RECENT_RING = 8, // the last windows, for the median need of a fast drop + EPISODE_RING = 32 // more than the HORIZON_MS / LAMBDA_MS + 1 the cost rule looks at + }; + + struct Episode + { + time_t start; + Int height; + }; + + Real m_hist[BUCKETS]; ///< The normal windows, forgetting. + Real m_mass; + Real m_timeMs; ///< The time the normal windows cover, forgetting the same way. + Real m_saved[BUCKETS]; ///< The baseline before a level shift was adopted, for the undo. + Real m_savedMass; + Real m_savedTimeMs; + Int m_savedBase; + Int m_undoMs; ///< Window time left in which a return to the saved baseline undoes the adopted level. + Int m_undoCalmMs; + Episode m_pending; ///< The episode an adoption came from, a spike after all if it is undone. + + Bool m_inSpike; + time_t m_spikeStart; + Int m_peakNeed; + Int m_episodeStallMs; + Int m_episodeMs; ///< Window time of the open episode after the window that opened it. + Int m_calmMs; + time_t m_sampleTime[SAMPLE_RING]; + Int m_sampleNeed[SAMPLE_RING]; + Int m_sampleMs[SAMPLE_RING]; + Int m_sampleCount; + + Episode m_episodes[EPISODE_RING]; + Int m_episodeCount; ///< Recorded so far, the ring holds the last EPISODE_RING. + Int m_lastCover; + Int m_recentNeed[RECENT_RING]; + Int m_recentMs[RECENT_RING]; + Int m_recentCount; + Int m_gateCleanMs; ///< Window time in a row the need stayed within the target + 1. + time_t m_gateClusterStart; ///< Start of the current spike cluster. + time_t m_gatePrevClusterStart; ///< Start of the cluster before it. + + static void forget(Real *hist, Real &mass, Real factor); + static void addSample(Real *hist, Real &mass, Int need); + static Int quantile(const Real *hist, Real mass, Real q); + Int baseline(Int runAhead, Int marginFrames) const; + void recordEpisode(const Episode &episode); +}; + // What the network overlay and the connection rating need to know about one slot beyond its ConnectionStats. struct NetworkOverlayInfo { NetworkOverlayInfo() : cushionSamples(0), cushionP5(0), cushionP50(0) , stallsWindow(0), stallsMinute(0), stalledNow(FALSE), waitingOnSlot(FALSE) - , slotInBandMs(-1), runAhead(0), frameGroupingMs(0) {} + , runAhead(0), frameGroupingMs(0) + , isRouter(FALSE), controllerOn(FALSE), marginFrames(1), changeDir(0), changeReason(CHANGE_NONE), changeAgoMs(-1) + , ctlReason(CTL_NONE), ctlMinSum(NO_PAIR_SUM), ctlPairA(-1), ctlPairB(-1), ctlBase(0), ctlCover(0), ctlSpikesPerMin10(0), ctlSpikeSizeMs(0), ctlInSpike(FALSE), ctlSinceChangeMs(0) {} + + enum ChangeReason { CHANGE_NONE, CHANGE_START, CHANGE_LAG_SPIKE, CHANGE_STALL, CHANGE_CLEAN }; + enum { NO_PAIR_SUM = 999 }; ///< ctlMinSum when no pair reported + enum ControllerReason { CTL_NONE, CTL_SEED, CTL_HOLD, CTL_BASELINE, CTL_COVER, CTL_UNCOVER, CTL_CONGESTION, CTL_OUTAGE, CTL_FORMULA }; Int cushionSamples; ///< Frames before execution that the frame commands of this slot arrived, over the stats window. Int cushionP5; @@ -183,8 +268,24 @@ struct NetworkOverlayInfo Int stallsMinute; ///< The same over the last minute. Bool stalledNow; ///< The local game is waiting for commands right now. Bool waitingOnSlot; ///< ... and this slot is one it is waiting on. - Int slotInBandMs; ///< In-band round trip this slot contributes to the run ahead, -1 if unknown. Int runAhead; ///< Frames. Int frameGroupingMs; ///< The send interval. + + Bool isRouter; ///< This machine decides the run ahead, the controller fields are valid only here. + Bool controllerOn; + Int marginFrames; ///< Cushion frames each direction of a pair keeps spare. + Int changeDir; ///< Direction of the last applied run ahead change, 1 up, -1 down, 0 none yet. + Int changeReason; ///< ChangeReason, known on every client. + Int changeAgoMs; ///< Since that change took effect, -1 if none. + Int ctlReason; ///< ControllerReason of the last decision. + Int ctlMinSum; ///< Lowest sum of the two cushion directions of a pair, NO_PAIR_SUM if no pair reported. + Int ctlPairA; + Int ctlPairB; + Int ctlBase; ///< Frames, the baseline quantile of the normal windows. + Int ctlCover; ///< Frames bought for spikes on top of the baseline. + Int ctlSpikesPerMin10; ///< Spike episodes per minute, times 10. + Int ctlSpikeSizeMs; ///< The median spike height over the baseline. + Bool ctlInSpike; ///< A spike episode is open, the run ahead holds. + Int ctlSinceChangeMs; }; diff --git a/Core/GameEngine/Include/GameNetwork/NetCommandMsg.h b/Core/GameEngine/Include/GameNetwork/NetCommandMsg.h index e8d297a9ac0..1959c49c5f5 100644 --- a/Core/GameEngine/Include/GameNetwork/NetCommandMsg.h +++ b/Core/GameEngine/Include/GameNetwork/NetCommandMsg.h @@ -312,11 +312,30 @@ class NetRunAheadMetricsCommandMsg : public NetCommandMsgT= 0 && isLinkUp(slot); +} + +// Input delay for a round trip: through the router round trip x (1 + slack); sent directly with the controller one way x (1 + slack) + send interval + margin. +Real ConnectionManager::estimateRunAheadMs(Real roundTripMs, const NetworkSettings::Send &sendSettings, const NetworkSettings::InputDelay &inputDelay, Bool oneWay) { + const Real slackScale = 1.0f + (Real)inputDelay.slack_percent / 100.0f; + if (!oneWay) { + return roundTripMs * slackScale; + } + const Real frameMs = 1000.0f / (Real)GENERALS_ONLINE_HIGH_FPS_LIMIT; + return roundTripMs * 0.5f * slackScale + (Real)sendSettings.frames_per_packet * frameMs + (Real)inputDelay.margin_frames * frameMs; +} + +Int ConnectionManager::getRunAheadCap() const { + return runAheadCap(m_networkSettings); +} + +Bool ConnectionManager::acceptsRunAhead(Int runAhead) const { + if (TheLAN != nullptr || TheNGMPGame == nullptr) { + return TRUE; + } + if (runAhead == 0) { + // the router runs a match without remote players at no run ahead + for (Int i = 0; i < MAX_SLOTS; ++i) { + if (i != m_localSlot && m_connections[i] != nullptr) { + return FALSE; + } + } + return TRUE; + } + return runAhead >= getRunAheadFloor() && runAhead <= getRunAheadCap(); +} + +Int ConnectionManager::computeStartRunAhead(Real roundTripMs, Bool oneWay) const { + // no repair budget at the start + const Int runAhead = (Int)ceilf(estimateRunAheadMs(roundTripMs, m_networkSettings.send, m_networkSettings.input_delay, oneWay) * (Real)GENERALS_ONLINE_HIGH_FPS_LIMIT / 1000.0f); + return clamp(getRunAheadFloor(), runAhead, getRunAheadCap()); +} + +void ConnectionManager::applyStartRunAhead(Int oldRunAhead, Int newRunAhead) { + if (newRunAhead < oldRunAhead) { + for (Int frame = newRunAhead; frame < oldRunAhead; ++frame) { + for (Int i = 0; i < MAX_SLOTS; ++i) { + if (m_frameData[i] != nullptr) { + m_frameData[i]->resetFrame((UnsignedInt)frame, FALSE); + } + } + } + } else if (newRunAhead > oldRunAhead) { + zeroFrames((UnsignedInt)oldRunAhead, (UnsignedInt)(newRunAhead - oldRunAhead)); + } +} + Int ConnectionManager::getRunAheadFloor() const { const NetworkSettings::InputDelay &runAhead = m_networkSettings.input_delay; if (runAhead.min_frames > 0) { @@ -681,33 +795,41 @@ static Int relayClass(NetCommandType type) { } } +// The mesh connection to a slot is up. +Bool ConnectionManager::isLinkUp(Int slot) { + if (TheNGMPGame == nullptr || slot < 0 || slot >= MAX_SLOTS) { + return FALSE; + } + NetworkMesh *mesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + const NGMPGameSlot *gameSlot = (const NGMPGameSlot *)TheNGMPGame->getConstSlot(slot); + PlayerConnection *connection = (mesh != nullptr && gameSlot != nullptr) ? mesh->GetConnectionForUser(gameSlot->m_userID) : nullptr; + return connection != nullptr && connection->GetState() == EConnectionState::CONNECTED_DIRECT; +} + // send.direct is on and every peer has a direct connection; commands then bypass the packet router. -Bool ConnectionManager::isDirectSendActive() { +Bool ConnectionManager::isDirectSendActive(Bool atStart) { if (TheNGMPGame == nullptr || !m_networkSettings.send.direct) { return FALSE; } const time_t now = timeGetTime(); - if (m_directSendCheckedAt != 0 && (now - m_directSendCheckedAt) < DIRECT_RECHECK_MS) { + if (!atStart && m_directSendCheckedAt != 0 && (now - m_directSendCheckedAt) < DIRECT_RECHECK_MS) { return m_directSendActive; } m_directSendCheckedAt = now; Bool meshUp = TRUE; - NetworkMesh *mesh = NGMP_OnlineServicesManager::GetNetworkMesh(); for (Int slot = 0; slot < MAX_SLOTS && meshUp; ++slot) { if (slot == m_localSlot || !isPeerActive(slot)) { continue; } #if !RETAIL_COMPATIBLE_NETWORKING - // a player that is out needs no commands - if (m_disconnectManager != nullptr && m_disconnectManager->isPlayerOutOfGame(slot, this)) { + // a player that is out needs no commands, before the match runs nobody is out + if (!atStart && m_disconnectManager != nullptr && m_disconnectManager->isPlayerOutOfGame(slot, this)) { continue; } #endif - const NGMPGameSlot *gameSlot = (const NGMPGameSlot *)TheNGMPGame->getConstSlot(slot); - PlayerConnection *connection = (mesh != nullptr && gameSlot != nullptr) ? mesh->GetConnectionForUser(gameSlot->m_userID) : nullptr; - if (connection == nullptr || connection->GetState() != EConnectionState::CONNECTED_DIRECT) { + if (!isLinkUp(slot)) { meshUp = FALSE; } } @@ -1264,6 +1386,9 @@ void ConnectionManager::doRelay() { Int senderSlot = -2; if (m_transport->m_inBuffer[i].senderKnown) { senderSlot = slotForUserID(m_transport->m_inBuffer[i].senderUserID); + if (senderSlot >= 0) { + m_lastHeardMs[senderSlot] = timeGetTime(); + } if (senderSlot < 0) { NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] dropped a packet of %d commands from user %lld, who is not in this game", (Int)packet->getNumCommands(), (long long)m_transport->m_inBuffer[i].senderUserID); } @@ -1582,6 +1707,20 @@ void ConnectionManager::processRunAheadMetrics(NetRunAheadMetricsCommandMsg *msg if (isPlayerConnected(playerID)) { m_latencyAverages[playerID] = msg->getAverageLatency(); m_fpsAverages[playerID] = msg->getAverageFps(); +#if defined(GENERALS_ONLINE) + if (timeGetTime() >= m_ctlIgnoreUntil) { + for (Int sender = 0; sender < MAX_SLOTS; ++sender) { + // NO_CUSHION is above any cushion, the lowest report of the window wins + m_ctlMin[playerID][sender] = min(m_ctlMin[playerID][sender], msg->getCushionMin(sender)); + m_ctlReportLate[playerID] += msg->getLate(sender); + } + m_ctlReportStall[playerID] += msg->getStallMs(); + m_ctlReportTime[playerID] = timeGetTime(); + if (msg->getScreenshot()) { + m_ctlScreenshotSeen = TRUE; + } + } +#endif //DEBUG_LOG(("ConnectionManager::processRunAheadMetrics - player %d, fps = %d, latency = %f", player, msg->getAverageFps(), msg->getAverageLatency())); // NGMP_CHANGE: Modern machines render games at much higher framerates, 100 is no longer only achievable when a game is in the background... @@ -1857,7 +1996,29 @@ void ConnectionManager::processFrameInfo(NetFrameCommandMsg *msg) { m_frameData[playerID]->setFrameCommandCount(msg->getExecutionFrame(), msg->getCommandCount()); #if defined(GENERALS_ONLINE) if (!isCopy && (playerID != (UnsignedInt)m_localSlot) && (TheGameLogic != nullptr)) { - m_cushionStats[playerID].note((Int)msg->getExecutionFrame() - (Int)TheGameLogic->getFrame(), timeGetTime()); + const Int cushion = (Int)msg->getExecutionFrame() - (Int)TheGameLogic->getFrame(); + const time_t cushionTime = timeGetTime(); + m_cushionStats[playerID].note(cushion, cushionTime); + // frame infos of one packet share the ms + enum { PACE_SAME_BURST_MS = 2, PACE_GAP_SAMPLE_CAP_MS = 100 }; // caps the gap sample + if (m_paceLastFreshMs[playerID] != 0 && (cushionTime - m_paceLastFreshMs[playerID]) >= PACE_SAME_BURST_MS) { + const Real gap = (Real)min(cushionTime - m_paceLastFreshMs[playerID], PACE_GAP_SAMPLE_CAP_MS); + if (m_paceGapAvgMs[playerID] <= 0.0f) { + m_paceGapAvgMs[playerID] = gap; + m_paceGapDevMs[playerID] = gap * 0.5f; + } else { + const Real deviation = (gap > m_paceGapAvgMs[playerID]) ? (gap - m_paceGapAvgMs[playerID]) : (m_paceGapAvgMs[playerID] - gap); + m_paceGapDevMs[playerID] += (deviation - m_paceGapDevMs[playerID]) * 0.25f; + m_paceGapAvgMs[playerID] += (gap - m_paceGapAvgMs[playerID]) * 0.125f; + } + } + m_paceLastFreshMs[playerID] = cushionTime; + if (msg->getExecutionFrame() >= m_ctlValidFromExec[playerID]) { + m_ctlWindowMin[playerID] = min(m_ctlWindowMin[playerID], cushion); + if (cushion <= 0) { + ++m_ctlLateCount[playerID]; + } + } } #endif } @@ -2298,11 +2459,15 @@ void ConnectionManager::update(Bool isInGame) { } void ConnectionManager::updateRunAhead(Int oldRunAhead, Int frameRate, Bool didSelfSlug, Int nextExecutionFrame) { - static time_t lasttimesent = 0; time_t curTime = timeGetTime(); #if defined(GENERALS_ONLINE) m_lastKnownRunAhead = oldRunAhead; + if (m_localSlot == m_packetRouterSlot) { + updateControllerOutage(curTime); + } else { + m_fastDropStep = FALSE; + } #endif #if defined(GENERALS_ONLINE) @@ -2310,7 +2475,13 @@ void ConnectionManager::updateRunAhead(Int oldRunAhead, Int frameRate, Bool didS #else const time_t decisionInterval = TheGlobalData->m_networkRunAheadMetricsTime; #endif - if ((lasttimesent == 0) || ((curTime - lasttimesent) > decisionInterval)) { +#if defined(GENERALS_ONLINE) + // after a fast drop step the next decision only waits the short dead time + const time_t gateInterval = (m_fastDropStep && FAST_DROP_DEAD_MS < decisionInterval) ? (time_t)FAST_DROP_DEAD_MS : decisionInterval; +#else + const time_t gateInterval = decisionInterval; +#endif + if ((m_lastRunAheadDecisionTime == 0) || ((curTime - m_lastRunAheadDecisionTime) > gateInterval)) { #if defined(GENERALS_ONLINE) if (m_networkSettings.input_delay.formula.latency_window_ms <= 0) { // the in-band latency is averaged over the samples since the previous decision @@ -2377,13 +2548,25 @@ void ConnectionManager::updateRunAhead(Int oldRunAhead, Int frameRate, Bool didS } newRunAhead += repairFrames; #endif +#if defined(GENERALS_ONLINE) + newRunAhead = clamp(minRunAheadForClamp, newRunAhead, runAheadCap(m_networkSettings)); +#else newRunAhead = clamp(minRunAheadForClamp, newRunAhead, MAX_FRAMES_AHEAD / 2); +#endif #if defined(GENERALS_ONLINE) + const Bool controllerOn = m_networkSettings.input_delay.controller; const Int decreaseAfter = m_networkSettings.input_delay.formula.decrease_after_decisions; // a command that was issued but has not taken effect yet is the run ahead to decide from, not the one still running const Int baseRunAhead = (m_pendingExecFrame != 0 && oldRunAhead != m_pendingRunAhead && TheGameLogic->getFrame() <= m_pendingExecFrame + 2) ? m_pendingRunAhead : oldRunAhead; - newRunAhead = applyDecreaseHold(baseRunAhead, newRunAhead, decreaseAfter); + if (controllerOn) + { + newRunAhead = decideControllerRunAhead(baseRunAhead, newRunAhead, minRunAheadForClamp, curTime, decreaseAfter); + } + else + { + newRunAhead = applyDecreaseHold(baseRunAhead, newRunAhead, decreaseAfter); + } #endif NetRunAheadCommandMsg* msg = newInstance(NetRunAheadCommandMsg); @@ -2488,6 +2671,21 @@ void ConnectionManager::updateRunAhead(Int oldRunAhead, Int frameRate, Bool didS msg->setID(GenerateNextCommandID()); } msg->setAverageLatency(m_frameMetrics.getAverageLatency()); +#if defined(GENERALS_ONLINE) + { + Int lowest[MAX_SLOTS]; + Int late[MAX_SLOTS]; + Int stallMs; + Bool screenshot; + collectControllerReport(lowest, late, stallMs, screenshot); + for (Int sender = 0; sender < MAX_SLOTS; ++sender) { + msg->setCushionMin(sender, lowest[sender]); + msg->setLate(sender, late[sender]); + } + msg->setStallMs(stallMs); + msg->setScreenshot(screenshot); + } +#endif // see above for explanation. // if (didSelfSlug) { @@ -2512,11 +2710,15 @@ void ConnectionManager::updateRunAhead(Int oldRunAhead, Int frameRate, Bool didS } msg->detach(); } - lasttimesent = curTime; + m_lastRunAheadDecisionTime = curTime; } } #if defined(GENERALS_ONLINE) +static Int framesToMs(Int frames) { + return (frames * 1000 + GENERALS_ONLINE_HIGH_FPS_LIMIT - 1) / GENERALS_ONLINE_HIGH_FPS_LIMIT; +} + // The run ahead goes up at once and falls only after decreaseAfter asks for less in a row, then to the highest of them. Int ConnectionManager::applyDecreaseHold(Int oldRunAhead, Int askedRunAhead, Int decreaseAfter) { if (m_runAheadStillInitial) { @@ -2536,14 +2738,370 @@ Int ConnectionManager::applyDecreaseHold(Int oldRunAhead, Int askedRunAhead, Int return askedRunAhead; } +void ConnectionManager::resetControllerWindow(Bool sinceChange) { + m_ctlScreenshotSeen = FALSE; + for (Int i = 0; i < MAX_SLOTS; ++i) { + m_ctlReportLate[i] = 0; + m_ctlReportStall[i] = 0; + if (sinceChange) { + m_ctlReportTime[i] = 0; + } + for (Int sender = 0; sender < MAX_SLOTS; ++sender) { + m_ctlMin[i][sender] = NetRunAheadMetricsCommandMsg::NO_CUSHION; + } + } +} + void ConnectionManager::noteRunAheadApplied(Int oldRunAhead, Int newRunAhead) { if (oldRunAhead == newRunAhead) { return; } + // what the overlay says about this change: the router knows the controller's reason, the others infer it from the direction and their own stalls + { + const time_t appliedNow = timeGetTime(); + Int changeReason = NetworkOverlayInfo::CHANGE_CLEAN; + if (m_runAheadStillInitial) { + changeReason = NetworkOverlayInfo::CHANGE_START; + } else if (newRunAhead > oldRunAhead) { + m_stallStats.update(appliedNow); + const Bool router = (m_localSlot == m_packetRouterSlot); + const Bool stalled = router ? (m_ctlLastReason == NetworkOverlayInfo::CTL_CONGESTION) : (m_stallStats.getCount(appliedNow, NETWORK_STATS_WINDOW_MS) > 0); + changeReason = stalled ? NetworkOverlayInfo::CHANGE_STALL : NetworkOverlayInfo::CHANGE_LAG_SPIKE; + } + m_changeDir = (newRunAhead > oldRunAhead) ? 1 : -1; + m_changeReason = changeReason; + m_changeAppliedTime = appliedNow; + static const char *const s_changeNames[] = { "none", "start", "lag spike", "stall", "clean" }; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[RUNAHEAD] input delay %d->%d ms (%s) reason=%s", framesToMs(oldRunAhead), framesToMs(newRunAhead), (m_changeDir > 0) ? "rising" : "lowering", s_changeNames[changeReason]); + } m_runAheadStillInitial = FALSE; + if (TheGameLogic == nullptr) { + return; + } + // commands issued from this frame on carry an execution frame of at least this + the new run ahead + const UnsignedInt validFrom = TheGameLogic->getFrame() + newRunAhead; + for (Int sender = 0; sender < MAX_SLOTS; ++sender) { + m_ctlValidFromExec[sender] = validFrom; + m_ctlWindowMin[sender] = NetRunAheadMetricsCommandMsg::NO_CUSHION; + m_ctlLateCount[sender] = 0; + } + m_ctlStallMs = 0; + m_ctlStallMark = timeGetTime(); +} + +void ConnectionManager::noteScreenshotCapture(Bool uploadFollows) { + // the capture blocks this frame, and the upload competes with the game traffic until it is done, a lost one times out + const time_t holdMs = uploadFollows ? 15000 : 3000; + m_ctlScreenshotUntil = timeGetTime() + holdMs; + m_screenshotStallUntil = m_ctlScreenshotUntil; +} + +void ConnectionManager::noteScreenshotDone() { + // the waits the upload left can end after it + m_ctlScreenshotUntil = timeGetTime() + 2000; + m_screenshotStallUntil = m_ctlScreenshotUntil; +} + +void ConnectionManager::collectControllerReport(Int *lowest, Int *late, Int &stallMs, Bool &screenshot) { + const time_t now = timeGetTime(); + for (Int sender = 0; sender < MAX_SLOTS; ++sender) { + lowest[sender] = m_ctlWindowMin[sender]; + m_ctlWindowMin[sender] = NetRunAheadMetricsCommandMsg::NO_CUSHION; + late[sender] = m_ctlLateCount[sender]; + m_ctlLateCount[sender] = 0; + } + if (m_stallActive && m_stallWaitedOnRemote && (now - m_stallStartTime) >= framesToMs(1)) { + m_ctlStallMs += (Int)(now - ((m_stallStartTime > m_ctlStallMark) ? m_stallStartTime : m_ctlStallMark)); + } + stallMs = m_ctlStallMs; + m_ctlStallMs = 0; + m_ctlStallMark = now; + // the report that ends the hold is still flagged + screenshot = m_ctlScreenshotUntil != 0; + if (screenshot && now >= m_ctlScreenshotUntil) { + m_ctlScreenshotUntil = 0; + } +} + +/** Takes the own report of the window and drops it, returns its stall ms. */ +Int ConnectionManager::discardControllerReport() { + Int lowest[MAX_SLOTS]; + Int late[MAX_SLOTS]; + Int stallMs; + Bool screenshot; + collectControllerReport(lowest, late, stallMs, screenshot); + return stallMs; +} + +/** A player silent for 1 s is an outage: the controller holds, drops its stalls and the open episode. */ +void ConnectionManager::updateControllerOutage(time_t now) { + enum { OUTAGE_MS = 1000 }; + Int silentSlot = -1; + time_t silentMs = 0; + for (Int slot = 0; slot < MAX_SLOTS; ++slot) { + if (slot != m_localSlot && isPlayerConnected(slot) && m_lastHeardMs[slot] != 0 && (now - m_lastHeardMs[slot]) >= OUTAGE_MS) { + if ((now - m_lastHeardMs[slot]) > silentMs) { + silentMs = now - m_lastHeardMs[slot]; + silentSlot = slot; + } + } + } + + if (silentSlot >= 0) { + if (m_ctlOutageSince == 0) { + m_ctlOutageSince = now - silentMs; + m_spikeController.dropEpisode(); + NetworkLog(ELogVerbosity::LOG_RELEASE, "[CONTROLLER] outage slot=%d silent for %dms, holding R %d", silentSlot, (Int)silentMs, m_lastKnownRunAhead); + } + return; + } + + if (m_ctlOutageSince != 0) { + const time_t outageMs = now - m_ctlOutageSince; + m_ctlOutageSince = 0; + const Int stallMs = discardControllerReport(); + resetControllerWindow(TRUE); + m_spikeController.dropEpisode(); + // reports still on their way cover the outage + const time_t ignoreUntil = now + getEffectiveDecisionIntervalMs(m_lastKnownRunAhead); + if (ignoreUntil > m_ctlIgnoreUntil) { + m_ctlIgnoreUntil = ignoreUntil; + } + m_ctlLastWindowTime = m_ctlIgnoreUntil; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[CONTROLLER] outage over after %dms, dropped its stalls (own %dms), the controller resumes", (Int)outageMs, stallMs); + } +} + +/** Router: run ahead from the lowest cushion sum of the pairs (plus margin) and the stalls, via RunAheadSpikeController. */ +Int ConnectionManager::decideControllerRunAhead(Int oldRunAhead, Int formulaRunAhead, Int floorRunAhead, time_t now, Int decreaseAfter) { + const NetworkSettings::InputDelay &config = m_networkSettings.input_delay; + const Int cap = runAheadCap(m_networkSettings); + m_fastDropStep = FALSE; + const Int margin = config.margin_frames; + const Int noCushion = NetRunAheadMetricsCommandMsg::NO_CUSHION; + const Real frameMs = 1000.0f / (Real)GENERALS_ONLINE_HIGH_FPS_LIMIT; + const time_t formulaAfterMs = 2 * NETWORK_STATS_WINDOW_MS; // no complete pair for this long, the latency formula takes over + + if (!m_ctlSeeded) { + if (getMaximumLatency() <= 0.0f && ++m_ctlSeedWait < 3) { + return oldRunAhead; + } + m_ctlSeeded = TRUE; + m_ctlLastPairTime = now; + m_spikeController.reset(); + resetControllerWindow(TRUE); + // sent directly the seed is the one way run ahead the controller settles at, the same estimate the start run ahead used + const Bool oneWay = usesOneWayRunAhead(); + // the probes of the players still here: the averaged latencies still hold the samples of a player that just left + const Real liveLatency = getLiveProbeLatency(); + const Real seedLatency = (liveLatency > 0.0f) ? liveLatency : getMaximumLatency(); + const Int seedRunAhead = oneWay ? computeStartRunAhead(seedLatency * 1000.0f, TRUE) : formulaRunAhead; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[CONTROLLER] R %d->%d reason=seed rtt_used=%.0fms one_way=%d", oldRunAhead, seedRunAhead, seedLatency * 1000.0f, (Int)oneWay); + m_ctlLastReason = NetworkOverlayInfo::CTL_SEED; + m_ctlChangeTime = now; + m_ctlIgnoreUntil = now + getEffectiveDecisionIntervalMs(seedRunAhead); + m_ctlLastWindowTime = m_ctlIgnoreUntil; + return seedRunAhead; + } + + // dead time: reports of the last change are still arriving + if (now < m_ctlIgnoreUntil) { + return oldRunAhead; + } + + // an outage is no reason to raise, the waits it causes are not the network's trend + if (m_ctlOutageSince != 0) { + const Int stallMs = discardControllerReport(); + resetControllerWindow(FALSE); + m_spikeController.dropEpisode(); + m_ctlLastWindowTime = now; + m_ctlLastReason = NetworkOverlayInfo::CTL_OUTAGE; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[CONTROLLER] R %d->%d reason=outage silent_for=%dms stallms_dropped=%d", oldRunAhead, oldRunAhead, (Int)(now - m_ctlOutageSince), stallMs); + return oldRunAhead; + } + + // the router's own row + { + Int ownLowest[MAX_SLOTS]; + Int ownLate[MAX_SLOTS]; + Int ownStallMs; + Bool ownScreenshot; + collectControllerReport(ownLowest, ownLate, ownStallMs, ownScreenshot); + if (ownScreenshot) { + m_ctlScreenshotSeen = TRUE; + } + for (Int sender = 0; sender < MAX_SLOTS; ++sender) { + m_ctlMin[m_localSlot][sender] = min(m_ctlMin[m_localSlot][sender], ownLowest[sender]); + m_ctlReportLate[m_localSlot] += ownLate[sender]; + } + m_ctlReportStall[m_localSlot] += ownStallMs; + } + + // wait for every player's first report since the change, 2 D at most + { + Bool allReported = TRUE; + for (Int a = 0; a < MAX_SLOTS; ++a) { + if (a != m_localSlot && isPlayerConnected(a) && m_ctlReportTime[a] == 0) { + allReported = FALSE; + } + } + if (!allReported && (now - m_ctlChangeTime) < 2 * getEffectiveDecisionIntervalMs(oldRunAhead)) { + return oldRunAhead; + } + } + + // the need of the window: the pair with the lowest sum of its two directions sets it + Int minSum = noCushion; + Int pairA = -1; + Int pairB = -1; + Int pairCount = 0; + Int late = 0; + Int stallMs = 0; + for (Int a = 0; a < MAX_SLOTS; ++a) { + if (a != m_localSlot && !isPlayerConnected(a)) { + continue; + } + late += m_ctlReportLate[a]; + stallMs = max(stallMs, m_ctlReportStall[a]); + for (Int b = a + 1; b < MAX_SLOTS; ++b) { + if (b != m_localSlot && !isPlayerConnected(b)) { + continue; + } + if (m_ctlMin[a][b] == noCushion || m_ctlMin[b][a] == noCushion) { + continue; + } + ++pairCount; + const Int sum = m_ctlMin[a][b] + m_ctlMin[b][a]; + if (sum < minSum) { + minSum = sum; + pairA = a; + pairB = b; + } + } + } + // a slow producer feeds every other player late at any run ahead, its own cushion piles up while the others wait for it + Int slowSlot = -1; + Int slowFps = 0; + Int slowCushion = noCushion; + { + const Int lagFrames = max(SLOW_PRODUCER_MIN_LAG_FRAMES, oldRunAhead / 4); + for (Int p = 0; p < MAX_SLOTS && slowSlot < 0; ++p) { + if (p != m_localSlot && !isPlayerConnected(p)) { + continue; + } + Int ownCushion = noCushion; + for (Int sender = 0; sender < MAX_SLOTS; ++sender) { + if (sender != p) { + ownCushion = min(ownCushion, m_ctlMin[p][sender]); + } + } + const Bool lowFps = m_fpsAverages[p] > 0 && m_fpsAverages[p] < SLOW_PRODUCER_FPS; + const Bool piledUp = ownCushion != noCushion && ownCushion > oldRunAhead + lagFrames; + if (lowFps || piledUp) { + slowSlot = p; + slowFps = m_fpsAverages[p]; + slowCushion = ownCushion; + } + } + } + const Bool screenshotInterval = m_ctlScreenshotSeen; + const Int windowMs = (Int)(now - m_ctlLastWindowTime); + m_ctlLastWindowTime = now; + resetControllerWindow(FALSE); + + if (slowSlot != m_ctlSlowSlot) { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[CONTROLLER] slow producer slot=%d (was %d) fps=%d own_cushion=%d R=%d, the stalls waiting for it (%dms) do not raise R", slowSlot, m_ctlSlowSlot, slowFps, (slowCushion == noCushion) ? -1 : slowCushion, oldRunAhead, stallMs); + m_ctlSlowSlot = slowSlot; + } + // no run ahead can buy back the wait for a player that is slower than the game + if (slowSlot >= 0) { + stallMs = 0; + } + + const Bool havePair = pairCount > 0; + const Int stallGateMs = framesToMs(3); // a wait of less than 3 F is not seen and one more frame of run ahead would not fix it + const Bool stalled = !screenshotInterval && stallMs >= stallGateMs; + // 2 R - sum frames are missing over the pair, half of it per direction, plus the margin each direction keeps + const Int twiceShort = 2 * oldRunAhead - minSum; + Int need = havePair ? (((twiceShort >= 0) ? ((twiceShort + 1) / 2) : -((-twiceShort) / 2)) + margin) : 0; + if (stalled) { + need = max(need, oldRunAhead + (Int)ceil(stallMs / frameMs) + margin); + } + if (havePair) { + m_ctlLastPairTime = now; + } + m_ctlLastMinSum = havePair ? minSum : NetworkOverlayInfo::NO_PAIR_SUM; + m_ctlLastPairA = pairA; + m_ctlLastPairB = pairB; + + Int newRunAhead = oldRunAhead; + const char *reason = "hold"; + Int reasonCode = NetworkOverlayInfo::CTL_HOLD; + Bool fastDrop = FALSE; + if (screenshotInterval) { + // every client captures at the same second, the window says nothing about the network + } else if (!havePair && !stalled) { + if ((now - m_ctlLastPairTime) >= formulaAfterMs) { + newRunAhead = applyDecreaseHold(oldRunAhead, formulaRunAhead, decreaseAfter); + reason = "formula"; + reasonCode = NetworkOverlayInfo::CTL_FORMULA; + } + } else { + // the largest round trip jitter of the mesh probes widens what counts as a spike + Real jitterMs = 0.0f; + NetworkMesh *pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + if (pMesh != nullptr) { + for (auto &kvPair : pMesh->GetAllConnections()) { + jitterMs = max(jitterMs, kvPair.second.m_probeStats.getJitter()); + } + } + const RunAheadSpikeController::Decision d = m_spikeController.onWindow(now, windowMs, need, stalled, stalled ? stallMs : 0, jitterMs, oldRunAhead, margin, frameMs, floorRunAhead, cap, config.fast_recovery, slowSlot >= 0); + static const char *const s_reasonNames[] = { "hold", "baseline", "cover", "uncover", "congestion" }; + static const Int s_reasonCodes[] = { NetworkOverlayInfo::CTL_HOLD, NetworkOverlayInfo::CTL_BASELINE, NetworkOverlayInfo::CTL_COVER, NetworkOverlayInfo::CTL_UNCOVER, NetworkOverlayInfo::CTL_CONGESTION }; + newRunAhead = d.runAhead; + fastDrop = d.fastDrop; + reason = s_reasonNames[d.reason]; + reasonCode = s_reasonCodes[d.reason]; + m_ctlLastBase = d.base; + m_ctlLastCover = d.cover; + m_ctlLastSpikesPerMin10 = d.spikesPerMin10; + m_ctlLastSpikeSizeMs = d.spikeSizeMs; + m_ctlLastInSpike = d.inSpike; + } + newRunAhead = clamp(floorRunAhead, newRunAhead, cap); + m_ctlLastReason = reasonCode; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[CONTROLLER] R %d->%d reason=%s need=%d minS=%d pair=%d-%d late=%d stallms=%d", + oldRunAhead, newRunAhead, reason, need, havePair ? minSum : (Int)NetworkOverlayInfo::NO_PAIR_SUM, pairA, pairB, late, stallMs); + + if (newRunAhead != oldRunAhead) { + resetControllerWindow(TRUE); + m_ctlChangeTime = now; + // a fast drop waits the shortest dead time before the next step down + m_fastDropStep = fastDrop && newRunAhead < oldRunAhead; + m_ctlIgnoreUntil = now + (m_fastDropStep ? FAST_DROP_DEAD_MS : getEffectiveDecisionIntervalMs(newRunAhead)); + m_ctlLastWindowTime = m_ctlIgnoreUntil; + } + return newRunAhead; } #endif +Real ConnectionManager::getLiveProbeLatency() +{ + NetworkMesh *pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + if (pMesh == nullptr) { + return 0.0f; + } + Int highestMs = 0; + for (auto &kvPair : pMesh->GetAllConnections()) { + const Int slot = slotForUserID(kvPair.first); + if (slot < 0 || slot == m_localSlot || !isPlayerConnected(slot)) { + continue; + } + highestMs = max(highestMs, kvPair.second.GetSeedLatency()); + } + return (Real)highestMs / 1000.0f; +} + // The average of the two highest in-band round trips, our own slot carries the one to our slowest peer. Real ConnectionManager::getMaximumLatency() { @@ -3858,12 +4416,61 @@ void ConnectionManager::noteStalling(Bool stalling, UnsignedInt frame) { } m_stallActive = TRUE; m_stallStartTime = now; - } else if (!stalling && m_stallActive) { + m_stallWaitedOnRemote = FALSE; + } + if (stalling) { + // only a remote player's missing commands are a network stall, a missing local slot or the disconnect manager holding the frame is not + for (Int i = 0; i < MAX_SLOTS && !m_stallWaitedOnRemote; ++i) { + if ((i != m_localSlot) && (m_frameData[i] != nullptr) && (m_frameData[i]->getIsQuitting() == FALSE) + && (m_frameData[i]->getFrameCommandCount(frame) != m_frameData[i]->getCommandCount(frame))) { + m_stallWaitedOnRemote = TRUE; + } + } + } else if (m_stallActive) { m_stallActive = FALSE; - m_stallStats.note((Int)(now - m_stallStartTime), m_stallMissingMask, now); + // a wait shorter than a frame is scheduler jitter, a frame is the least a larger run ahead could buy back + if (m_stallWaitedOnRemote && (now - m_stallStartTime) >= framesToMs(1)) { + m_ctlStallMs += (Int)(now - ((m_stallStartTime > m_ctlStallMark) ? m_stallStartTime : m_ctlStallMark)); + } + // the overlay, the rating and the logs count the stalls the controller counts: a wait of a frame or more for a + // remote player's commands, not one a local screenshot capture caused + const Bool screenshotStall = (m_screenshotStallUntil != 0 && m_stallStartTime < m_screenshotStallUntil); + if (m_stallWaitedOnRemote && (now - m_stallStartTime) >= framesToMs(1) && !screenshotStall) { + m_stallStats.note((Int)(now - m_stallStartTime), m_stallMissingMask, now); + } } } +/** Frames after this one that every remote player's commands are complete for (lowest, up to maxFrames); overdue = its next frame info is later than its usual gap. */ +Int ConnectionManager::getRemoteLookAhead(UnsignedInt frame, Int maxFrames, Bool &overdue) { + const time_t now = timeGetTime(); + Int lowest = -1; + overdue = FALSE; + for (Int i = 0; i < MAX_SLOTS; ++i) { + if ((i == m_localSlot) || (m_frameData[i] == nullptr) || m_frameData[i]->getIsQuitting()) { + continue; + } + Int ahead = 0; + while ((ahead < maxFrames) && (m_frameData[i]->getFrameCommandCount(frame + 1 + ahead) == m_frameData[i]->getCommandCount(frame + 1 + ahead))) { + ++ahead; + } + // a usual gap plus twice its deviation, never less than a frame + Bool late = FALSE; + if (m_paceLastFreshMs[i] != 0 && m_paceGapAvgMs[i] > 0.0f) { + Real limitMs = m_paceGapAvgMs[i] + 2.0f * m_paceGapDevMs[i]; + if (limitMs < (Real)framesToMs(1)) { + limitMs = (Real)framesToMs(1); + } + late = (Real)(now - m_paceLastFreshMs[i]) > limitMs; + } + if ((lowest < 0) || (ahead < lowest) || ((ahead == lowest) && late && !overdue)) { + lowest = ahead; + overdue = late; + } + } + return lowest; +} + Bool ConnectionManager::getOverlayInfo(Int slot, NetworkOverlayInfo &info) { time_t now = timeGetTime(); m_stallStats.update(now); @@ -3872,14 +4479,28 @@ Bool ConnectionManager::getOverlayInfo(Int slot, NetworkOverlayInfo &info) { info.stalledNow = m_stallActive; info.runAhead = m_lastKnownRunAhead; info.frameGroupingMs = m_frameGroupingMs; + const NetworkSettings::InputDelay &runAheadConfig = m_networkSettings.input_delay; + info.controllerOn = runAheadConfig.controller; + info.marginFrames = runAheadConfig.margin_frames; + info.isRouter = (m_localSlot == m_packetRouterSlot); + info.changeDir = m_changeDir; + info.changeReason = m_changeReason; + info.changeAgoMs = (m_changeAppliedTime != 0) ? (Int)(now - m_changeAppliedTime) : -1; + info.ctlReason = m_ctlLastReason; + info.ctlMinSum = m_ctlLastMinSum; + info.ctlPairA = m_ctlLastPairA; + info.ctlPairB = m_ctlLastPairB; + info.ctlBase = m_ctlLastBase; + info.ctlCover = m_ctlLastCover; + info.ctlSpikesPerMin10 = m_ctlLastSpikesPerMin10; + info.ctlSpikeSizeMs = m_ctlLastSpikeSizeMs; + info.ctlInSpike = m_ctlLastInSpike; + info.ctlSinceChangeMs = (m_ctlChangeTime != 0) ? (Int)(now - m_ctlChangeTime) : 0; if (slot >= 0 && slot < MAX_SLOTS) { info.stallsWindow = m_stallStats.getCountForSlot(slot, now, NETWORK_STATS_WINDOW_MS); info.stallsMinute = m_stallStats.getCountForSlot(slot, now, 60000); info.waitingOnSlot = m_stallActive && ((m_stallMissingMask & (1u << slot)) != 0); - if (isPlayerConnected(slot)) { - info.slotInBandMs = (Int)(m_latencyAverages[slot] * 1000.0f); - } m_cushionStats[slot].update(now); info.cushionSamples = m_cushionStats[slot].getSampleCount(); info.cushionP5 = m_cushionStats[slot].getPercentile(5); diff --git a/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp b/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp index 129659295a2..3606167395f 100644 --- a/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp +++ b/Core/GameEngine/Source/GameNetwork/ConnectionStats.cpp @@ -348,3 +348,375 @@ Int StallStats::getCountForSlot(Int slot, time_t now, Int windowMs) const } return count; } + +// Rules run on window time, so any decision interval behaves the same. +static const Int SPIKE_LAMBDA_MS = 10000; // a player gives 1 frame of permanent delay for 1/lambda less freeze, the cover pays for spikes recurring faster than this +static const Int SPIKE_FORGET_MS = 500; +static const Real SPIKE_FORGET = 0.983f; // per SPIKE_FORGET_MS of window time, about a 30 s memory: up after 1.5 s of high windows, down after 90 s +static const Int SPIKE_WINDOW_MIN_MS = 100; +static const Int SPIKE_WINDOW_MAX_MS = 5000; // a decision interval is 2 x 128 frames at most, a longer window counts as this much +static const Int SPIKE_HEAD_MIN_FRAMES = 4; // over the baseline that opens an episode, above the normal tail +static const Int SPIKE_HEAD_JITTER_FACTOR = 3; // or 3 x the round trip jitter when that is more +static const Int SPIKE_TAIL_FRAMES = 1; // over the baseline that counts as calm again +static const Int SPIKE_TAIL_MS = 1000; // calm window time in a row that ends an episode +static const Int SPIKE_SUSTAIN_MS = 2000; // episode window time after its first window that makes it a level shift +static const Int SPIKE_UNDO_MS = SPIKE_SUSTAIN_MS; // window time after an adoption in which a return to the old baseline undoes it +static const Int SPIKE_WARMUP_MS = SPIKE_SUSTAIN_MS; // normal window time before the quantile is trusted, the seed stands until then +static const Int SPIKE_HORIZON_MS = 120000; // the spike rate memory, 3 periods of a 34 s spike +static const Int SPIKE_ALLOWED = SPIKE_HORIZON_MS / SPIKE_LAMBDA_MS; // spikes taller than the cover allowed in the horizon, a rate of 1 / lambda +static const Int SPIKE_EXTRA_MAX_FRAMES = 12; // the cover buys 200 ms at most, a taller spike is not affordable +static const Int SAFE_CLEAN_MS = 3000; // window time the need stays within the target + 1 before a fast drop is safe +static const Int SAFE_HISTORY_MS = 120000; // a fast drop needs the current spike to be the only one in this much time before it +static const Int SPIKE_DOWN_STEP_FRAMES = 2; // per decision going down, at least: the engine drains the queued frames without a stall, the step keeps the delay change gradual +static const Int SPIKE_MAX_RAISE_FRAMES = 24; // most a single decision raises while a slow producer is suspected, loss stalls raise without a bound + +// Adds value to an ascending array of count entries. +static void insertSorted(Int *array, Int &count, Int value) +{ + Int pos = count; + while (pos > 0 && array[pos - 1] > value) + { + array[pos] = array[pos - 1]; + --pos; + } + array[pos] = value; + ++count; +} + +RunAheadSpikeController::RunAheadSpikeController() +{ + reset(); +} + +void RunAheadSpikeController::reset() +{ + for (Int bucket = 0; bucket < BUCKETS; ++bucket) + { + m_hist[bucket] = 0.0f; + m_saved[bucket] = 0.0f; + } + m_mass = 0.0f; + m_timeMs = 0.0f; + m_savedMass = 0.0f; + m_savedTimeMs = 0.0f; + m_savedBase = 0; + m_undoMs = 0; + m_undoCalmMs = 0; + m_pending.start = 0; + m_pending.height = 0; + m_inSpike = FALSE; + m_spikeStart = 0; + m_peakNeed = 0; + m_episodeStallMs = 0; + m_episodeMs = 0; + m_calmMs = 0; + m_sampleCount = 0; + m_episodeCount = 0; + m_lastCover = 0; + m_recentCount = 0; + m_gateCleanMs = 0; + m_gateClusterStart = 0; + m_gatePrevClusterStart = 0; +} + +void RunAheadSpikeController::dropEpisode() +{ + m_inSpike = FALSE; + m_episodeMs = 0; + m_calmMs = 0; + m_sampleCount = 0; + m_undoMs = 0; + m_gateCleanMs = 0; +} + +void RunAheadSpikeController::forget(Real *hist, Real &mass, Real factor) +{ + for (Int bucket = 0; bucket < BUCKETS; ++bucket) + { + hist[bucket] *= factor; + } + mass *= factor; +} + +void RunAheadSpikeController::addSample(Real *hist, Real &mass, Int need) +{ + hist[clamp(0, need, BUCKETS - 1)] += 1.0f; + mass += 1.0f; +} + +Int RunAheadSpikeController::quantile(const Real *hist, Real mass, Real q) +{ + const Real wanted = q * mass; + Real sum = 0.0f; + for (Int bucket = 0; bucket < BUCKETS; ++bucket) + { + sum += hist[bucket]; + if (sum >= wanted && sum > 0.0f) + { + return bucket; + } + } + return BUCKETS - 1; +} + +Int RunAheadSpikeController::baseline(Int runAhead, Int marginFrames) const +{ + if (m_timeMs < SPIKE_WARMUP_MS) + { + return runAhead - marginFrames; + } + // q = 1 - W / lambda + const Real q = 1.0f - m_timeMs / (m_mass * (Real)SPIKE_LAMBDA_MS); + return quantile(m_hist, m_mass, q); +} + +void RunAheadSpikeController::recordEpisode(const Episode &episode) +{ + m_episodes[m_episodeCount % EPISODE_RING] = episode; + ++m_episodeCount; +} + +RunAheadSpikeController::Decision RunAheadSpikeController::onWindow(time_t now, Int windowMs, Int need, Bool stalled, Int stallMs, Real jitterMs, Int runAhead, Int marginFrames, Real frameMs, Int floorFrames, Int capFrames, Bool safeFastDrop, Bool boundRaise) +{ + Decision d; + d.runAhead = runAhead; + d.reason = REASON_HOLD; + d.fastDrop = FALSE; + + const Int window = clamp(SPIKE_WINDOW_MIN_MS, windowMs, SPIKE_WINDOW_MAX_MS); + const Real factor = (Real)pow((double)SPIKE_FORGET, (double)window / (double)SPIKE_FORGET_MS); + need = clamp(0, need, BUCKETS - 1); + m_recentNeed[m_recentCount % RECENT_RING] = need; + m_recentMs[m_recentCount % RECENT_RING] = window; + ++m_recentCount; + + // every window ages the histograms by the time it covers, spike windows too + forget(m_hist, m_mass, factor); + m_timeMs *= factor; + if (m_undoMs > 0) + { + forget(m_saved, m_savedMass, factor); + m_savedTimeMs *= factor; + } + + Int base = baseline(runAhead, marginFrames); + const Int head = max(SPIKE_HEAD_MIN_FRAMES, (Int)ceil(SPIKE_HEAD_JITTER_FACTOR * jitterMs / frameMs)); + Bool undone = FALSE; + + // an adopted level the need does not stay at was a spike after all, the old baseline comes back at once + if (m_undoMs > 0) + { + m_undoMs -= window; + if (need <= m_savedBase + SPIKE_TAIL_FRAMES && !stalled) + { + m_undoCalmMs += window; + if (m_undoCalmMs >= SPIKE_TAIL_MS) + { + for (Int restore = 0; restore < BUCKETS; ++restore) + { + m_hist[restore] = m_saved[restore]; + } + m_mass = m_savedMass; + m_timeMs = m_savedTimeMs; + m_undoMs = 0; + undone = TRUE; + d.reason = REASON_BASELINE; + recordEpisode(m_pending); + base = baseline(runAhead, marginFrames); + } + } + else + { + m_undoCalmMs = 0; + } + } + + if (!m_inSpike) + { + // only normal windows feed the baseline + if (need > base + head || stalled) + { + m_inSpike = TRUE; + m_spikeStart = now; + // a spike that opens before SAFE_CLEAN_MS of calm belongs to the same cluster of loss + if (m_gateClusterStart == 0 || m_gateCleanMs >= SAFE_CLEAN_MS) + { + m_gatePrevClusterStart = m_gateClusterStart; + m_gateClusterStart = now; + } + m_peakNeed = need; + m_episodeStallMs = stallMs; + m_episodeMs = 0; + m_calmMs = 0; + m_sampleTime[0] = now; + m_sampleNeed[0] = need; + m_sampleMs[0] = window; + m_sampleCount = 1; + } + else + { + addSample(m_hist, m_mass, need); + m_timeMs += window; + } + } + else + { + m_peakNeed = max(m_peakNeed, need); + m_episodeStallMs += stallMs; + m_episodeMs += window; + m_sampleTime[m_sampleCount % SAMPLE_RING] = now; + m_sampleNeed[m_sampleCount % SAMPLE_RING] = need; + m_sampleMs[m_sampleCount % SAMPLE_RING] = window; + ++m_sampleCount; + m_calmMs = (need <= base + SPIKE_TAIL_FRAMES && !stalled) ? (m_calmMs + window) : 0; + // the height a run ahead would need to cover the whole episode, its stalls split over windows add up + Episode episode; + episode.start = m_spikeStart; + episode.height = max(0, max(m_peakNeed, runAhead + marginFrames + (Int)ceil(m_episodeStallMs / frameMs)) - base); + if (m_calmMs >= SPIKE_TAIL_MS) + { + recordEpisode(episode); + m_inSpike = FALSE; + addSample(m_hist, m_mass, need); + m_timeMs += window; + } + else if (m_episodeMs >= SPIKE_SUSTAIN_MS) + { + // level shift: rebase on the median need of the last SUSTAIN + Int recent[SAMPLE_RING]; + Int recentCount = 0; + Int recentMs = 0; + const Int kept = min(m_sampleCount, SAMPLE_RING); + for (Int k = 0; k < kept; ++k) + { + const Int at = (m_sampleCount - 1 - k) % SAMPLE_RING; + if ((now - m_sampleTime[at]) < (time_t)SPIKE_SUSTAIN_MS) + { + insertSorted(recent, recentCount, m_sampleNeed[at]); + recentMs += m_sampleMs[at]; + } + } + const Int level = (recentCount > 0) ? recent[recentCount / 2] : need; + for (Int save = 0; save < BUCKETS; ++save) + { + m_saved[save] = m_hist[save]; + m_hist[save] = 0.0f; + } + m_savedMass = m_mass; + m_savedTimeMs = m_timeMs; + m_savedBase = base; + m_mass = 0.0f; + m_pending = episode; + for (Int fill = 0; fill < max(recentCount, 1); ++fill) + { + addSample(m_hist, m_mass, level); + } + m_timeMs = (Real)max(recentMs, SPIKE_WARMUP_MS); + m_undoMs = SPIKE_UNDO_MS; + m_undoCalmMs = 0; + m_inSpike = FALSE; + d.reason = REASON_CONGESTION; + } + } + + const Bool warm = m_timeMs >= SPIKE_WARMUP_MS; + base = baseline(runAhead, marginFrames); + + // cover = the tallest spike height that more than horizon / lambda episodes reach + Int heights[EPISODE_RING]; + Int inHorizon = 0; + const Int stored = min(m_episodeCount, EPISODE_RING); + for (Int e = 0; e < stored; ++e) + { + const Episode &episode = m_episodes[e]; + if ((now - episode.start) <= SPIKE_HORIZON_MS) + { + Int pos = inHorizon; + while (pos > 0 && heights[pos - 1] < episode.height) + { + heights[pos] = heights[pos - 1]; + --pos; + } + heights[pos] = episode.height; + ++inHorizon; + } + } + const Int cover = (inHorizon > SPIKE_ALLOWED) ? min(SPIKE_EXTRA_MAX_FRAMES, heights[SPIKE_ALLOWED]) : 0; + const Int target = clamp(floorFrames, base + cover, capFrames); + + m_gateCleanMs = (!m_inSpike && !stalled && need <= target + 1) ? (m_gateCleanMs + window) : 0; + const Bool gateClean = m_gateCleanMs >= SAFE_CLEAN_MS; + const Bool gateIsolated = m_gatePrevClusterStart == 0 || (m_gateClusterStart - m_gatePrevClusterStart) > SAFE_HISTORY_MS; + + // fast drop: target the recent median need + Int goal = target; + Bool fastDrop = FALSE; + // the gate stays open through our own steps down: they do not touch the clean counter and the reseeded baseline only moves the target + const Bool gateOpen = safeFastDrop && gateClean && gateIsolated && !m_inSpike && !undone && warm; + if (gateOpen) + { + Int recent[RECENT_RING]; + Int recentCount = 0; + Int recentMs = 0; + const Int kept = min(m_recentCount, RECENT_RING); + for (Int k = 0; k < kept && recentMs < SAFE_CLEAN_MS; ++k) + { + const Int at = (m_recentCount - 1 - k) % RECENT_RING; + insertSorted(recent, recentCount, m_recentNeed[at]); + recentMs += m_recentMs[at]; + } + const Int level = clamp(floorFrames, recent[recentCount / 2], capFrames); + if (level < target) + { + goal = level; + fastDrop = TRUE; + } + } + + // no change during a spike, a raise decided now arrives after it ended + if (!m_inSpike && warm) + { + if (target > runAhead) + { + d.runAhead = boundRaise ? min(target, runAhead + SPIKE_MAX_RAISE_FRAMES) : target; + if (d.reason != REASON_CONGESTION) + { + d.reason = (cover > m_lastCover) ? REASON_COVER : REASON_BASELINE; + } + } + else if (goal < runAhead) + { + // fast recovery: a third of the excess per step, never below the goal; with the safe gate open the goal is the recent need, + // the baseline is reseeded from it and the next step does not wait the long dead time + const Int step = safeFastDrop ? max(SPIKE_DOWN_STEP_FRAMES, (runAhead - goal + 2) / 3) : SPIKE_DOWN_STEP_FRAMES; + if (gateOpen) + { + d.fastDrop = TRUE; + } + if (fastDrop) + { + for (Int reseed = 0; reseed < BUCKETS; ++reseed) + { + m_hist[reseed] = 0.0f; + } + m_mass = 0.0f; + addSample(m_hist, m_mass, goal); + m_timeMs = (Real)SPIKE_WARMUP_MS; + } + d.runAhead = undone ? target : max(goal, runAhead - step); + if (d.reason != REASON_CONGESTION && !undone) + { + d.reason = (cover < m_lastCover) ? REASON_UNCOVER : REASON_BASELINE; + } + } + } + m_lastCover = cover; + + const Int sizeFrames = (inHorizon > 0) ? heights[inHorizon / 2] : 0; + d.base = base; + d.cover = cover; + d.spikesPerMin10 = inHorizon * 10 * 60000 / SPIKE_HORIZON_MS; + d.spikeSizeMs = (Int)(sizeFrames * frameMs + 0.5f); + d.inSpike = m_inSpike; + return d; +} diff --git a/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp b/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp index 68b2c5941a7..ebc21d51b75 100644 --- a/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp +++ b/Core/GameEngine/Source/GameNetwork/FrameMetrics.cpp @@ -100,15 +100,21 @@ void FrameMetrics::init() { NetworkMesh* pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); if (TheNGMPGame != nullptr && pMesh != nullptr) { + // average ping of the connections that have one, else the retail 200 ms int totalLatency = 0; + int latencyCount = 0; std::map& connections = pMesh->GetAllConnections(); for (auto& kvPair : connections) { - PlayerConnection& conn = kvPair.second; - totalLatency += conn.GetLatency(); + const int latency = kvPair.second.GetLatency(); + if (latency >= 0) + { + totalLatency += latency; + ++latencyCount; + } } - m_averageLatency = (Real)((Real)totalLatency / 1000.f) / (Real)connections.size(); + m_averageLatency = (latencyCount > 0) ? (Real)totalLatency / 1000.f / (Real)latencyCount : (Real)0.2; } else { diff --git a/Core/GameEngine/Source/GameNetwork/NetCommandMsg.cpp b/Core/GameEngine/Source/GameNetwork/NetCommandMsg.cpp index 2dead32b164..a314b9caa64 100644 --- a/Core/GameEngine/Source/GameNetwork/NetCommandMsg.cpp +++ b/Core/GameEngine/Source/GameNetwork/NetCommandMsg.cpp @@ -447,6 +447,14 @@ NetCommandMsg::Select NetPlayerLeaveCommandMsg::getSmallNetPacketSelect() const NetRunAheadMetricsCommandMsg::NetRunAheadMetricsCommandMsg() { m_averageLatency = 0.0; m_averageFps = 0; +#if defined(GENERALS_ONLINE) + for (Int slot = 0; slot < MAX_SLOTS; ++slot) { + m_cushionMin[slot] = NO_CUSHION; + m_late[slot] = 0; + } + m_stallMs = 0; + m_screenshot = FALSE; +#endif m_commandType = NETCOMMANDTYPE_RUNAHEADMETRICS; } diff --git a/Core/GameEngine/Source/GameNetwork/NetPacketStructs.cpp b/Core/GameEngine/Source/GameNetwork/NetPacketStructs.cpp index 846d1443738..f6a70272371 100644 --- a/Core/GameEngine/Source/GameNetwork/NetPacketStructs.cpp +++ b/Core/GameEngine/Source/GameNetwork/NetPacketStructs.cpp @@ -392,6 +392,14 @@ size_t NetPacketRunAheadMetricsCommandData::copyBytes(UnsignedByte *buffer, cons FixedData data; data.averageLatency = cmdMsg->getAverageLatency(); data.averageFps = cmdMsg->getAverageFps(); +#if defined(GENERALS_ONLINE) + for (Int slot = 0; slot < MAX_SLOTS; ++slot) { + const Int lowest = cmdMsg->getCushionMin(slot); + data.cushionMin[slot] = (lowest == CommandMsg::NO_CUSHION) ? (Byte)-128 : (Byte)clamp(-127, lowest, 127); + data.late[slot] = (UnsignedByte)clamp(0, cmdMsg->getLate(slot), 255); + } + data.stallMs = (UnsignedShort)clamp(0, cmdMsg->getStallMs(), 0x7FFF) | (cmdMsg->getScreenshot() ? 0x8000 : 0); +#endif return network::writeObject(buffer, data); } @@ -402,10 +410,25 @@ size_t NetPacketRunAheadMetricsCommandData::readMessage(NetCommandRef &ref, NetP FixedData data; data.averageLatency = 0.2f; data.averageFps = 30; +#if defined(GENERALS_ONLINE) + for (Int slot = 0; slot < MAX_SLOTS; ++slot) { + data.cushionMin[slot] = (Byte)-128; + data.late[slot] = 0; + } + data.stallMs = 0; +#endif size_t size = network::readObject(data, buf); cmdMsg->setAverageLatency(data.averageLatency); cmdMsg->setAverageFps(data.averageFps); +#if defined(GENERALS_ONLINE) + for (Int slot = 0; slot < MAX_SLOTS; ++slot) { + cmdMsg->setCushionMin(slot, (data.cushionMin[slot] == -128) ? (Int)CommandMsg::NO_CUSHION : (Int)data.cushionMin[slot]); + cmdMsg->setLate(slot, data.late[slot]); + } + cmdMsg->setStallMs(data.stallMs & 0x7FFF); + cmdMsg->setScreenshot((data.stallMs & 0x8000) != 0); +#endif return size; } diff --git a/Core/GameEngine/Source/GameNetwork/Network.cpp b/Core/GameEngine/Source/GameNetwork/Network.cpp index 2b22d47510b..5e07c3f8743 100644 --- a/Core/GameEngine/Source/GameNetwork/Network.cpp +++ b/Core/GameEngine/Source/GameNetwork/Network.cpp @@ -84,6 +84,9 @@ struct ConnectionMessage #pragma pack(pop) static const int CmdMsgLen = 6; //< Minimum size of a command packet (Int + Unsigned Short) +#if defined(GENERALS_ONLINE) +static const time_t START_RUNAHEAD_TIMEOUT_MS = 15000; //< A router that stays connected has this long after the load to send the start run ahead. +#endif // PRIVATE DATA /////////////////////////////////////////////////////////////// @@ -117,6 +120,8 @@ class Network : public NetworkInterface virtual void SeedLatencyData(int latency); virtual void requireMatchKeys() override { m_matchKeysRequired = TRUE; } virtual Bool matchKeysRequired() const override { return m_matchKeysRequired; } + virtual void setStartRunAheadRequired(Bool required) override { m_startRunAheadRequired = required; } + virtual void applyStartRunAhead(Int runAhead, Int maxRttMs, Int pairs) override; #endif void setLocalAddress(UnsignedInt ip, UnsignedInt port); @@ -256,7 +261,15 @@ class Network : public NetworkInterface Bool m_matchKeysRequired; Bool m_matchStartFailed; Bool startPrerequisitesReady() const; - void failMatchStart(); ///< Leaves rather than starting without authentication. + Bool m_startRunAheadRequired; ///< The router sends the start run ahead, the load is not complete until it arrived. + Bool m_startRunAheadApplied; + Bool m_loggedRejectedRunAhead; ///< A run ahead outside the floor and cap was ignored and logged. + Bool m_loadCompletePending; ///< The load finished before all start prerequisites arrived. + time_t m_loadCompletePendingSince; + void failMatchStart(); ///< Leaves rather than starting without authentication or the router decision. + + void updatePace(); ///< Sets the speed of the next frame from how far ahead the remote commands are. + Real m_paceSpeed; ///< Share of the full frame rate the next frame runs at, 1 = full. #endif }; @@ -376,7 +389,11 @@ void Network::init() m_lastFrame = 0; m_runAhead = min(max(30, MIN_RUNAHEAD), MAX_FRAMES_AHEAD / 2); ///< @todo: don't hard-code the run-ahead. +#if defined(GENERALS_ONLINE_HIGH_FPS_SERVER) + m_frameRate = GENERALS_ONLINE_HIGH_FPS_LIMIT; // the logic rate update() sets once the match runs +#else m_frameRate = 30; +#endif m_lastExecutionFrame = m_runAhead - 1; // subtract 1 since we're starting on frame 0 m_lastFrameCompleted = m_runAhead - 1; // subtract 1 since we're starting on frame 0 m_frameDataReady = FALSE; @@ -391,6 +408,12 @@ void Network::init() m_rateSampleMs = 0; m_rateSampleFrame = 0; m_measuredLogicRate = 0; + m_startRunAheadRequired = FALSE; + m_startRunAheadApplied = FALSE; + m_loggedRejectedRunAhead = FALSE; + m_loadCompletePending = FALSE; + m_loadCompletePendingSince = 0; + m_paceSpeed = 1.0f; #endif QueryPerformanceFrequency((LARGE_INTEGER*)&m_perfCountFreq); @@ -421,11 +444,39 @@ void Network::init() #if defined(GENERALS_ONLINE) void Network::SeedLatencyData(int highestLatency) { - NetworkLog(ELogVerbosity::LOG_RELEASE, "[PRESEED] Seeding with highest latency %d", highestLatency); + NetworkLog(ELogVerbosity::LOG_RELEASE, "[PRESEED] Seeding with the lobby round trip %d ms (p75 of the probes)", highestLatency); m_conMgr->SeedLatencyData(highestLatency); } +void Network::applyStartRunAhead(Int runAhead, Int maxRttMs, Int pairs) +{ + if (m_conMgr == nullptr || m_startRunAheadApplied || m_localStatus == NETLOCALSTATUS_POSTGAME) + { + return; + } + + runAhead = clamp(m_conMgr->getRunAheadFloor(), runAhead, m_conMgr->getRunAheadCap()); + const Int previousRunAhead = m_runAhead; + + // no frame of the match has run and no peer sent commands for one, every peer waits for our load complete first + m_conMgr->applyStartRunAhead(previousRunAhead, runAhead); + m_runAhead = runAhead; + m_lastExecutionFrame = m_runAhead - 1; + m_lastFrameCompleted = m_runAhead - 1; + if (maxRttMs > 0) + { + m_conMgr->SeedLatencyData(maxRttMs); + } + m_startRunAheadApplied = TRUE; + + NetworkLog(ELogVerbosity::LOG_RELEASE, "[RUNAHEAD] start R=%d (pairs=%d, max rtt=%dms)", m_runAhead, pairs, maxRttMs); + + if (m_loadCompletePending) + { + loadProgressComplete(); + } +} #endif #if defined(GENERALS_ONLINE) @@ -585,6 +636,11 @@ Bool Network::processCommand(GameMessage* msg) #endif #if defined(GENERALS_ONLINE) // Pregame updates drive loading; only entering gameplay requires completed setup. + // A load that was forced on because the router went silent has no start run ahead, every player that got here starts at the retail one. + if (m_startRunAheadRequired && !m_startRunAheadApplied) { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[RUNAHEAD] the load was forced on without the router's decision, start R=%d as in retail", (Int)m_runAhead); + applyStartRunAhead(m_runAhead, 0, 0); + } if (!startPrerequisitesReady()) { failMatchStart(); return TRUE; @@ -742,6 +798,13 @@ void Network::processFrameSynchronizedNetCommand(NetCommandRef* msg) { void Network::processRunAheadCommand(NetRunAheadCommandMsg* msg) { #if defined(GENERALS_ONLINE) + if (!m_conMgr->acceptsRunAhead(msg->getRunAhead())) { + if (!m_loggedRejectedRunAhead) { + m_loggedRejectedRunAhead = TRUE; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] ignored a run ahead of %d, outside %d..%d", (Int)msg->getRunAhead(), m_conMgr->getRunAheadFloor(), m_conMgr->getRunAheadCap()); + } + return; + } const Int previousRunAhead = m_runAhead; #endif m_runAhead = msg->getRunAhead(); @@ -891,6 +954,14 @@ void Network::liteupdate() { if (m_matchStartFailed) { return; } + if (m_loadCompletePending) { + if (startPrerequisitesReady()) { + loadProgressComplete(); + } else if ((timeGetTime() - m_loadCompletePendingSince) >= START_RUNAHEAD_TIMEOUT_MS) { + failMatchStart(); + return; + } + } #endif #if defined(RTS_DEBUG) @@ -936,14 +1007,16 @@ Bool Network::timeForNewFrame() { * to avoid being frozen by spikes in network lag. This will happen if another user's computer is * running too far behind us, so we need to slow down to let them catch up. */ - if (m_conMgr != NULL) { - Real cushion = m_conMgr->getMinimumCushion(); #if defined(GENERALS_ONLINE) - // the controller keeps a margin of one frame, slow down when the commands of the slowest player arrive less than that early - Real runAheadPercentage = 1.0f; + // stretch the next frame while a remote player's commands are short + if (m_conMgr != NULL && curTime >= m_nextFrameTime) { + updatePace(); + frameDelay = (__int64)((double)frameDelay / (double)m_paceSpeed); + } #else + if (m_conMgr != NULL) { + Real cushion = m_conMgr->getMinimumCushion(); Real runAheadPercentage = m_runAhead * (TheGlobalData->m_networkRunAheadSlack / (Real)100.0); // If we are at least 50% into our slack, we need to slow down. -#endif if (cushion < runAheadPercentage) { __int64 oldFrameDelay = frameDelay; frameDelay += oldFrameDelay / 10; // temporarily decrease the frame rate by 20%. @@ -953,6 +1026,7 @@ Bool Network::timeForNewFrame() { // DEBUG_LOG(("Average cushion = %f, run ahead percentage = %f", cushion, runAheadPercentage)); } } +#endif // Check to see if we can run another frame. if (curTime >= m_nextFrameTime) { @@ -966,7 +1040,12 @@ Bool Network::timeForNewFrame() { #endif if (m_nextFrameTime + catchUpFrames * frameDelay < curTime) { // If we get too far behind on our framerate we need to reset the nextFrameTime thing. +#if defined(GENERALS_ONLINE) + // after a stall the lost time is not caught up + m_nextFrameTime = curTime + frameDelay; +#else m_nextFrameTime = curTime; +#endif NetworkLog(ELogVerbosity::LOG_DEBUG, "Initializing m_nextFrameTime to %I64d", m_nextFrameTime); } else { // Set the soonest possible starting time for the next frame. @@ -980,6 +1059,37 @@ Bool Network::timeForNewFrame() { return FALSE; } +#if defined(GENERALS_ONLINE) +/** Slows the next frame while a remote player's commands run short; 90% floor, ramps down over 4 frames and up over 10. */ +void Network::updatePace() { + static const Real PACE_FLOOR_SPEED = 0.9f; + enum { PACE_SLOW_FRAMES = 4, PACE_RECOVER_FRAMES = 10 }; + + const Int margin = m_conMgr->getNetworkSettings().input_delay.margin_frames; + const Int lowFrames = margin + 1; + + Bool overdue = FALSE; + const Int ahead = m_conMgr->getRemoteLookAhead(TheGameLogic->getFrame(), lowFrames, overdue); + Int shortFrames = 0; + if (ahead >= 0) { + if (overdue) { + shortFrames = lowFrames - ahead; + } else if (ahead == 0) { + shortFrames = 1; + } + } + shortFrames = clamp(0, shortFrames, lowFrames); + + const Real share = (1.0f - PACE_FLOOR_SPEED) / (Real)lowFrames; + const Real target = 1.0f - share * (Real)shortFrames; + if (target < m_paceSpeed) { + m_paceSpeed = max(target, m_paceSpeed - (1.0f - PACE_FLOOR_SPEED) / (Real)PACE_SLOW_FRAMES); + } else { + m_paceSpeed = min(target, m_paceSpeed + (1.0f - PACE_FLOOR_SPEED) / (Real)PACE_RECOVER_FRAMES); + } +} +#endif + /** * Returns true if the game commands for the next frame have been put on the command list. */ @@ -1159,7 +1269,7 @@ void Network::updateLoadProgress(Int percent) #if defined(GENERALS_ONLINE) Bool Network::startPrerequisitesReady() const { - if (m_matchStartFailed) { + if (m_matchStartFailed || (m_startRunAheadRequired && !m_startRunAheadApplied)) { return FALSE; } if (m_matchKeysRequired) { @@ -1175,7 +1285,8 @@ void Network::failMatchStart() return; } NetworkMesh* mesh = NGMP_OnlineServicesManager::GetNetworkMesh(); - NetworkLog(ELogVerbosity::LOG_RELEASE, "[START] Missing match keys, leaving (keys=%zu/%zu)", + NetworkLog(ELogVerbosity::LOG_RELEASE, "[START] Missing start run ahead of a connected router, leaving (router=%d, keys=%zu/%zu)", + !m_startRunAheadRequired || m_startRunAheadApplied, mesh != nullptr ? mesh->GetMatchKeys().GetPeerKeyCount() : 0, mesh != nullptr ? mesh->GetMatchKeys().GetExpectedPeers() : 0); m_matchStartFailed = TRUE; @@ -1192,6 +1303,16 @@ void Network::loadProgressComplete() if (m_matchStartFailed) { return; } + // load complete waits for the peer keys and the start run ahead of the router + if (!startPrerequisitesReady()) { + if (!m_loadCompletePending) { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[START] Load complete, waiting for authenticated peers and router decision"); + m_loadCompletePendingSince = timeGetTime(); + } + m_loadCompletePending = TRUE; + return; + } + m_loadCompletePending = FALSE; #endif if (m_conMgr != nullptr) { m_conMgr->loadProgressComplete(); diff --git a/GeneralsMD/Code/GameEngine/Include/GameLogic/GameLogic.h b/GeneralsMD/Code/GameEngine/Include/GameLogic/GameLogic.h index 51c3b6c9e30..1e6dd6f4b26 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameLogic/GameLogic.h +++ b/GeneralsMD/Code/GameEngine/Include/GameLogic/GameLogic.h @@ -174,6 +174,7 @@ class GameLogic : public SubsystemInterface, public Snapshot void updateLoadProgress( Int progress ); void deleteLoadScreen(); + void revealFromLoadScreen(Bool loadingSaveGame); ///< Fades the load screen out and deletes it. //Kris: Cut setGameLoading() and replaced with setLoadingMap() and setLoadingSave() -- reason: nomenclature //void setGameLoading( Bool loading ) { m_loadingScene = loading; } diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/ConnectionRating.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/ConnectionRating.h index 1225735f622..98da8897293 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/ConnectionRating.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/ConnectionRating.h @@ -7,6 +7,7 @@ // It is the worse of two axes: delay (the input delay the link forces, in ms) and stability (stalls per minute). // 5 bars: delay <= 150 ms and no stall in the last minute; 1 bar: delay above 500 ms or 6 or more stalls per minute. // Colors: 4-5 green, 2-3 amber, 1 red. Red needs one of (RedReason): blackout, loss, delay or stalls, see RedReason in the .cpp. +// The delay is ConnectionManager::estimateRunAheadMs of the link's p75 probe round trip, the command round trip in a match without enough probes. // Stalls a healthy link does not explain wait on the peer's frame rate: slow PC, at most 3 bars and never red for it. // Too few samples (10 probes, 20 command acks) give no rating. class ConnectionRating @@ -44,9 +45,9 @@ class ConnectionRating }; /// lossPercent is the packet loss from the packet feedback, negative to use the loss of the stats. - static Inputs InputsFromProbe(const ConnectionStats &probe, Int slackPercent, Int sendIntervalMs, Real lossPercent); + static Inputs InputsFromProbe(const ConnectionStats &probe, Real lossPercent, Real expectedDelayMs); - static Inputs InputsFromMatch(const ConnectionStats &cmd, const NetworkOverlayInfo &info, Int slackPercent, Int sendIntervalMs, Real lossPercent); + static Inputs InputsFromMatch(const ConnectionStats &cmd, const NetworkOverlayInfo &info, Real lossPercent, Real expectedDelayMs); /// Fewer bars show at once, more only after the rating asked for them for a few seconds. static void Update(State &state, const Inputs &inputs, UnsignedInt nowMs); diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h index 57da3030398..9e580f97404 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NGMP_include.h @@ -12,7 +12,8 @@ enum class ENetworkChannel : BYTE NETWORK_CHANNEL_GAME = 0, NETWORK_CHANNEL_AC = 1, NETWORK_CHANNEL_PING = 2, - NETWORK_CHANNEL_KEYS = 3 // match signing keys, sent reliably + NETWORK_CHANNEL_KEYS = 3, // match signing keys, sent reliably + NETWORK_CHANNEL_START = 4 // lobby round trips to the router and its start run ahead back, sent reliably at match start }; #define CHECK_MAIN_THREAD assert(std::this_thread::get_id() == NGMP_OnlineServicesManager::g_MainThreadID); diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h index d370e440bc5..1247f33e724 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/NetworkMesh.h @@ -89,6 +89,7 @@ class PlayerConnection return static_cast(m_probeStats.getLatencyPercentile(k_seedPercentile) + 0.5f); } static constexpr int k_seedMinSamples = 10; // the probes a lobby round trip, a jitter or a rating needs, 2.5 s at the lobby interval + int GetMatchRatingMinSamples() const; ///< Half the probes the stats window holds at the match interval, one lost probe must not drop the probe p75. static constexpr int k_seedPercentile = 75; float GetSustainedLossPercent(int windowMs); ///< The worse direction of the packet feedback loss from at least 3 reports, -1 if unknown. @@ -249,6 +250,11 @@ class NetworkMesh return highestLatency; } + // Start run ahead: players send lobby round trips to the router, which sends frame 1's run ahead to all. + void BeginStartRunAhead(int routerSlot); ///< Call after the network parsed the user list. + void OnStartRunAheadMessage(int64_t connectionUserID, const uint8_t* data, size_t len); + bool BeginSyncedStart(int stage); ///< Call when the local part of the stage is done; false if there is nobody to start with. + bool SyncedStartDone(int stage); ///< Polled while waiting: true once every player is ready and the common leave time came, or on timeout. int SendGamePacket(void* pBuffer, uint32_t totalDataSize, int64_t userID); @@ -316,6 +322,66 @@ class NetworkMesh }; std::map m_keySync; ///< Per peer, when the key and the held keys were last sent. + struct StartRunAheadReport + { + uint64_t matchID = 0; + std::vector> rtts; ///< user and p75 round trip in ms of each connection of the reporter + bool directSend = false; ///< the reporter sends its commands directly to every peer, else through the router + }; + std::map m_startReports; ///< by reporter, kept until a match with another id begins + uint64_t m_startMatchID = 0; + bool m_startActive = false; ///< BeginStartRunAhead ran for m_startMatchID + bool m_startIsRouter = false; + bool m_startDecided = false; ///< the router sent its decision, or a peer applied the router's + int64_t m_startBeginMs = 0; + int64_t m_startRouterUserID = -1; + std::vector m_startPlayers; ///< user ids of the other human players of the match + int64_t m_startLocalUserID = -1; + struct StartDecision + { + uint64_t matchID = 0; + int runAhead = 0; + int maxRtt = 0; + int pairs = 0; + }; + std::map m_startDecisions; ///< by sender, a decision can come before this client began; only the router's counts + bool StartReportsComplete() const; + bool IsStartPlayer(int64_t userID) const; + bool IsStartPlayerLost(int64_t userID, int64_t nowMs); ///< No usable connection to the user for START_LOST_GRACE_MS + std::map m_startLostSinceMs; ///< by user, when its connection was first seen unusable + void PruneLostStartPlayers(int64_t nowMs); ///< The router stops waiting for the reports of players that left + void FallBackToRetailStart(); ///< The router left before it decided, every client that has no decision starts at the retail run ahead + void SanitizeStartReport(int64_t reporter, StartRunAheadReport& report) const; ///< keeps the entries that name a start player, clamps their round trips + void TickStartRunAhead(int64_t nowMs); + void DecideStartRunAhead(); + void ApplyStartDecision(); + void SendStartMessage(int64_t userID, const std::vector& payload); + // the stages of the synced start: 0 on the load screen, 1 when all the local setup is done + enum { SYNC_STAGES = 2 }; + struct StartReady + { + uint64_t matchID = 0; + bool direct = false; ///< the sender's direct path to every peer is up, or it waited long enough for it + }; + struct SyncStage + { + std::map ready; ///< by sender, can come before this client is ready + uint64_t goMatchID = 0; ///< the router's go, valid for m_startMatchID only + int goDelayMs = 0; + int64_t goReceivedMs = 0; + int64_t goFromUserID = -1; ///< a go before the router is known is kept only if the router sent it + bool hasGo = false; + bool active = false; + bool readySent = false; + int64_t beginMs = 0; + int64_t allInMs = 0; ///< when the last ready came in, 0 while one is missing + int64_t leaveAtMs = 0; ///< the common leave time once known, 0 before + }; + SyncStage m_sync[SYNC_STAGES]; + void SendSyncReady(int stage, int64_t nowMs, bool direct); + int CountSyncMissing(int stage, int64_t nowMs, int& total); ///< start players still to be heard from, players that left do not count + void FinishSyncedStart(int stage, const char* szReason, int64_t nowMs); + std::string m_strTurnUsername; std::string m_strTurnToken; std::string m_strTurnUsernameString; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index 60bb3e255d0..d5b11e182c1 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -436,7 +436,11 @@ inline void ReadServiceConfig(const nlohmann::json& j, ServiceConfig& c) if (j.contains("input_delay")) { const nlohmann::json& s = j["input_delay"]; + ReadServiceConfigValue(s, "controller", c.input_delay.controller); + ReadServiceConfigValue(s, "margin_frames", c.input_delay.margin_frames); ReadServiceConfigValue(s, "min_frames", c.input_delay.min_frames); + ReadServiceConfigValue(s, "max_frames", c.input_delay.max_frames); + ReadServiceConfigValue(s, "fast_recovery", c.input_delay.fast_recovery); ReadServiceConfigValue(s, "decision_interval_ms", c.input_delay.decision_interval_ms); ReadServiceConfigValue(s, "fps_history_samples", c.input_delay.fps_history_samples); ReadServiceConfigValue(s, "catch_up_frames", c.input_delay.catch_up_frames); @@ -483,6 +487,7 @@ inline void ClampServiceConfig(ServiceConfig& c) c.input_delay.formula.decrease_after_decisions = std::clamp(c.input_delay.formula.decrease_after_decisions, 0, 20); c.input_delay.catch_up_frames = std::clamp(c.input_delay.catch_up_frames, 1, 4); c.input_delay.min_frames = std::clamp(c.input_delay.min_frames, 0, 30); // the start run ahead is at least 30, a higher floor would make configs disagree + c.input_delay.max_frames = std::clamp(c.input_delay.max_frames, (std::max)(30, c.input_delay.min_frames), 128); c.input_delay.slack_percent = std::clamp(c.input_delay.slack_percent, 0, 200); c.input_delay.decision_interval_ms = std::clamp(c.input_delay.decision_interval_ms, 100, 60000); c.input_delay.fps_history_samples = std::clamp(c.input_delay.fps_history_samples, 1, 128); @@ -491,6 +496,7 @@ inline void ClampServiceConfig(ServiceConfig& c) c.input_delay.formula.latency_window_ms = std::clamp(c.input_delay.formula.latency_window_ms, 30 * 1000 / GENERALS_ONLINE_HIGH_FPS_LIMIT + 1, 60000); } c.send.frames_per_packet = std::clamp(c.send.frames_per_packet, 1, 16); + c.input_delay.margin_frames = std::clamp(c.input_delay.margin_frames, 0, 10); c.disconnect.freeze_before_screen_ms = std::clamp(c.disconnect.freeze_before_screen_ms, 1000, 30000); c.disconnect.remove_after_ms = std::clamp(c.disconnect.remove_after_ms, 10000, 180000); diff --git a/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp b/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp index 84890487da5..fc3f6521f9e 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GUICallbacks/Menus/WOLGameSetupMenu.cpp @@ -538,7 +538,12 @@ static void playerTooltip(GameWindow *window, bIsConnected = false; } connectionBars = pConnection->ComputeConnectionRating(); - connectionLatency = pConnection->GetLatency(); + // the same values the rating uses + connectionLatency = pConnection->GetSeedLatency(); + if (connectionLatency < 0) + { + connectionLatency = pConnection->GetLatency(); + } connectionJitter = pConnection->GetJitter(); const float sustainedLoss = pConnection->GetSustainedLossPercent(60000); connectionLossPct = (sustainedLoss >= 0.0f) ? static_cast(sustainedLoss + 0.5f) : -1; diff --git a/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp b/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp index 65ab0d87426..60d101fb948 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameClient/InGameUI.cpp @@ -7181,9 +7181,10 @@ namespace const Int k_playerViewTop = 500; const Real k_devViewTopFraction = 0.10f; // the dev table sits higher to clear the control bar + const Int k_changeShownMs = 10000; const Int k_nameMaxChars = 14; const Int k_devViewHeaderLines = 2; - const Int k_devViewFooterLines = 1; + const Int k_devViewFooterLines = 3; const wchar_t *const s_overlayHeaders[COL_COUNT] = { @@ -7191,6 +7192,8 @@ namespace L"Cush5/50", L"n", L"Stalls", L"Depth", L"OutL%", L"InL%" }; + const wchar_t *const s_overlayChangeNames[] = { L"", L"start", L"lag spike", L"stall", L"clean" }; + const wchar_t *const s_overlayControllerNames[] = { L"n/a", L"seed", L"hold", L"baseline", L"cover", L"uncover", L"congestion", L"outage", L"formula" }; struct OverlayRow { @@ -7505,7 +7508,7 @@ void InGameUI::drawNetworkOverlay(Int mode) if (row.info.cushionSamples > 0) { cell[COL_CUSHION].format(L"%d/%d", row.info.cushionP5, row.info.cushionP50); - sev[COL_CUSHION] = (row.info.cushionP5 <= 0) ? ConnectionRating::SEVERITY_BAD : ConnectionRating::SEVERITY_OK; + sev[COL_CUSHION] = (row.info.cushionP5 <= 0) ? ConnectionRating::SEVERITY_BAD : ((row.info.cushionP5 <= row.info.marginFrames) ? ConnectionRating::SEVERITY_WARN : ConnectionRating::SEVERITY_OK); } else { @@ -7563,6 +7566,28 @@ void InGameUI::drawNetworkOverlay(Int mode) text.format(L"Input delay %d ms, logic rate %u Hz", delayMs, cache.logicRate); x += overlayDrawText(k_strHeaderBase + 0, x, y, text, header.stalledNow ? overlayColor(ConnectionRating::SEVERITY_BAD) : neutral, dropColor) + segmentGap; + const Bool moving = (header.changeAgoMs >= 0 && header.changeAgoMs < k_changeShownMs); + if (moving && header.changeDir > 0) + { + text = L"rising"; + x += overlayDrawText(k_strHeaderBase + 1, x, y, text, overlayColor(ConnectionRating::SEVERITY_WARN), dropColor) + segmentGap / 2; + } + else if (moving && header.changeDir < 0) + { + text = L"lowering"; + x += overlayDrawText(k_strHeaderBase + 1, x, y, text, GameMakeColor(120, 220, 120, 255), dropColor) + segmentGap / 2; + } + else + { + text = L"steady"; + x += overlayDrawText(k_strHeaderBase + 1, x, y, text, dim, dropColor) + segmentGap / 2; + } + if (header.changeReason > NetworkOverlayInfo::CHANGE_NONE) + { + text.format(L"(%ls)", s_overlayChangeNames[header.changeReason]); + x += overlayDrawText(k_strHeaderBase + 2, x, y, text, dim, dropColor) + segmentGap; + } + if (header.stalledNow && !waitingOn.isEmpty()) { text.format(L"waiting on: %ls", waitingOn.str()); @@ -7709,9 +7734,54 @@ void InGameUI::drawNetworkOverlay(Int mode) y += lineH / 2; { + Int x = left; UnicodeString text; text.format(L"R %d frames (%d ms) send interval %d ms", header.runAhead, delayMs, header.frameGroupingMs); - overlayDrawText(k_strFooterBase + 0, left, y, text, neutral, dropColor); + x += overlayDrawText(k_strFooterBase + 0, x, y, text, neutral, dropColor) + segmentGap; + + if (header.isRouter && header.controllerOn) + { + const Int reasonSeverity = (header.ctlReason == NetworkOverlayInfo::CTL_CONGESTION || header.ctlReason == NetworkOverlayInfo::CTL_OUTAGE) ? ConnectionRating::SEVERITY_BAD : ((header.ctlReason == NetworkOverlayInfo::CTL_COVER || header.ctlInSpike) ? ConnectionRating::SEVERITY_WARN : ConnectionRating::SEVERITY_OK); + text.format(L"reason %ls", s_overlayControllerNames[header.ctlReason]); + x += overlayDrawText(k_strFooterBase + 1, x, y, text, overlayColor(reasonSeverity), dropColor) + segmentGap; + + const Int minSumNeeded = 2 * header.marginFrames; + if (header.ctlMinSum < NetworkOverlayInfo::NO_PAIR_SUM) + { + text.format(L"minS %d of %d (pair %d-%d)", header.ctlMinSum, minSumNeeded, header.ctlPairA, header.ctlPairB); + x += overlayDrawText(k_strFooterBase + 2, x, y, text, overlayColor(header.ctlMinSum <= 0 ? ConnectionRating::SEVERITY_BAD : (header.ctlMinSum < minSumNeeded ? ConnectionRating::SEVERITY_WARN : ConnectionRating::SEVERITY_OK)), dropColor) + segmentGap; + } + else + { + text = L"minS - (no pair)"; + x += overlayDrawText(k_strFooterBase + 2, x, y, text, dim, dropColor) + segmentGap; + } + if (header.ctlReason == NetworkOverlayInfo::CTL_NONE) + { + text = L"base n/a cover n/a"; + } + else + { + text.format(L"base %d cover %d%ls", header.ctlBase, header.ctlCover, header.ctlInSpike ? L" in spike" : L""); + } + overlayDrawText(k_strFooterBase + 3, x, y, text, neutral, dropColor); + y += lineH; + + text.format(L"spikes %d.%d/min size %d ms since_change %d ms", header.ctlSpikesPerMin10 / 10, header.ctlSpikesPerMin10 % 10, header.ctlSpikeSizeMs, header.ctlSinceChangeMs); + overlayDrawText(k_strFooterBase + 4, left, y, text, neutral, dropColor); + y += lineH; + } + else + { + if (header.changeReason > NetworkOverlayInfo::CHANGE_NONE) + { + text.format(L"last change %ls %d s ago", s_overlayChangeNames[header.changeReason], header.changeAgoMs / 1000); + x += overlayDrawText(k_strFooterBase + 5, x, y, text, neutral, dropColor) + segmentGap; + } + text = header.isRouter ? L"controller off" : L"host decides"; + overlayDrawText(k_strFooterBase + 6, x, y, text, dim, dropColor); + y += lineH; + } } #endif } diff --git a/GeneralsMD/Code/GameEngine/Source/GameLogic/System/GameLogic.cpp b/GeneralsMD/Code/GameEngine/Source/GameLogic/System/GameLogic.cpp index 1fa00d1b2e6..de6aefdebe6 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameLogic/System/GameLogic.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameLogic/System/GameLogic.cpp @@ -1193,6 +1193,43 @@ void GameLogic::updateLoadProgress(Int progress) } } +void GameLogic::revealFromLoadScreen(Bool loadingSaveGame) +{ + // if we're in a load game, don't fade yet + if (loadingSaveGame == FALSE && TheTransitionHandler != NULL && m_loadScreen) + { + TheFramePacer->reset(); + TheTransitionHandler->setGroup("FadeWholeScreen"); + while (!TheTransitionHandler->isFinished()) + { + TheWindowManager->update(); + if (!TheTransitionHandler->isFinished()) + { + TheDisplay->draw(); + setFPMode(); + TheFramePacer->update(); + } + + } + } + + if (m_loadScreen) + { + TheMouse->setVisibility(TRUE); + + /* + // + // delete load screen only when not loading a save game, for save games we still + // have more work to do and the load screen will be deleted elsewhere after + // we're all done with the load game progress + // + if( loadingSaveGame == FALSE ) + */ + deleteLoadScreen(); + + } +} + // ------------------------------------------------------------------------------------------------ /** Delete the load screen */ // ------------------------------------------------------------------------------------------------ @@ -2355,6 +2392,9 @@ void GameLogic::tryStartNewGame( Bool loadingSaveGame ) TheNetwork->loadProgressComplete(); TheNetwork->liteupdate(); } +#if defined(GENERALS_ONLINE) + const UnsignedInt loadCompleteTime = timeGetTime(); +#endif #if defined(GENERALS_ONLINE) // set a minimum display time for the load screen on GO to allow players to read ELO, army, stats, etc. @@ -2371,38 +2411,29 @@ void GameLogic::tryStartNewGame( Bool loadingSaveGame ) Sleep(100); } - // if we're in a load game, don't fade yet - if (loadingSaveGame == FALSE && TheTransitionHandler != NULL && m_loadScreen) + Bool syncedStart = FALSE; +#if defined(GENERALS_ONLINE) + // stay on the load screen until every player is ready to run frames, so nobody runs ahead into a stall + if (isInInternetGame() && !m_forceGameStartByTimeOut) { - TheFramePacer->reset(); - TheTransitionHandler->setGroup("FadeWholeScreen"); - while (!TheTransitionHandler->isFinished()) + NetworkMesh* pStartMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + if (pStartMesh != nullptr && pStartMesh->BeginSyncedStart(0)) { - TheWindowManager->update(); - if (!TheTransitionHandler->isFinished()) + syncedStart = TRUE; + while (!pStartMesh->SyncedStartDone(0)) { - TheDisplay->draw(); - setFPMode(); - TheFramePacer->update(); + updateLoadProgress(101); + Sleep(5); } - } + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SYNC] left the load screen %u ms after the load was complete", timeGetTime() - loadCompleteTime); } +#endif - if (m_loadScreen) + // with a synced start the load screen stays up until the other players are ready + if (!syncedStart) { - TheMouse->setVisibility(TRUE); - - /* - // - // delete load screen only when not loading a save game, for save games we still - // have more work to do and the load screen will be deleted elsewhere after - // we're all done with the load game progress - // - if( loadingSaveGame == FALSE ) - */ - deleteLoadScreen(); - + revealFromLoadScreen(loadingSaveGame); } #ifdef DUMP_PERF_STATS @@ -2508,7 +2539,7 @@ void GameLogic::tryStartNewGame( Bool loadingSaveGame ) { HideControlBar(); } - else + else if (!syncedStart) { ShowControlBar(FALSE); } @@ -2517,7 +2548,10 @@ void GameLogic::tryStartNewGame( Bool loadingSaveGame ) // Turn off the UI HideControlBar(); #endif - TheWritableGlobalData->m_loadScreenRender = FALSE; ///< mark to resume rendering as normal + if (!syncedStart) + { + TheWritableGlobalData->m_loadScreenRender = FALSE; ///< mark to resume rendering as normal + } // if we're in a gamespy game, mark us as playing if (TheGameSpyBuddyMessageQueue && TheGameSpyGame && isInInternetGame()) @@ -2543,6 +2577,28 @@ void GameLogic::tryStartNewGame( Bool loadingSaveGame ) } } + // the local setup took a different time on every machine, start the first frames together while the load screen is still up +#if defined(GENERALS_ONLINE) + if (syncedStart) + { + NetworkMesh* pStartMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + if (pStartMesh != nullptr && pStartMesh->BeginSyncedStart(1)) + { + while (!pStartMesh->SyncedStartDone(1)) + { + updateLoadProgress(101); + Sleep(2); + } + } + } +#endif + if (syncedStart) + { + revealFromLoadScreen(loadingSaveGame); + ShowControlBar(FALSE); + TheWritableGlobalData->m_loadScreenRender = FALSE; ///< mark to resume rendering as normal + } + //ReAllows quit menu to work during loading scene //setGameLoading(FALSE); setLoadingMap(FALSE); diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/ConnectionRating.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/ConnectionRating.cpp index 397c129250e..bcdc176f877 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/ConnectionRating.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/ConnectionRating.cpp @@ -35,30 +35,23 @@ static Real PredictStallsPerMinute(const ConnectionStats &stats, Real lossPercen return 2.1f * loss * 110.0f / (stats.getLatency() + 60.0f) + 0.1f * excess; } -static Real EstimateDelayMs(Real roundTripMs, Int slackPercent, Int sendIntervalMs) -{ - return roundTripMs * (1.0f + slackPercent / 100.0f) + sendIntervalMs; -} - ConnectionRating::Inputs::Inputs() : delayMs(0.0f), stallsPerMinute(0.0f), measuredStallsPerMinute(-1.0f), lossPercent(-1.0f), silentMs(-1), silentLimitMs(k_redSilentMs), stallInWindow(FALSE), slowPc(FALSE) { } -ConnectionRating::Inputs ConnectionRating::InputsFromProbe(const ConnectionStats &probe, Int slackPercent, Int sendIntervalMs, Real lossPercent) +ConnectionRating::Inputs ConnectionRating::InputsFromProbe(const ConnectionStats &probe, Real lossPercent, Real expectedDelayMs) { Inputs inputs; - // the p75 round trip, not the smoothed one a few spikes pull up - inputs.delayMs = EstimateDelayMs(probe.getLatencyPercentile(75), slackPercent, sendIntervalMs); + inputs.delayMs = expectedDelayMs; inputs.stallsPerMinute = PredictStallsPerMinute(probe, lossPercent); return inputs; } -ConnectionRating::Inputs ConnectionRating::InputsFromMatch(const ConnectionStats &cmd, const NetworkOverlayInfo &info, Int slackPercent, Int sendIntervalMs, Real lossPercent) +ConnectionRating::Inputs ConnectionRating::InputsFromMatch(const ConnectionStats &cmd, const NetworkOverlayInfo &info, Real lossPercent, Real expectedDelayMs) { Inputs inputs; - const Real roundTripMs = (info.slotInBandMs >= 0) ? (Real)info.slotInBandMs : cmd.getLatency(); - inputs.delayMs = EstimateDelayMs(roundTripMs, slackPercent, sendIntervalMs); + inputs.delayMs = expectedDelayMs; Real stallsPerMinute = PredictStallsPerMinute(cmd, lossPercent); if ((Real)info.stallsMinute >= k_slowPcMinStalls && stallsPerMinute < k_slowPcMaxLinkStalls) diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp index 62826ec555e..b2bf95adfb8 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NGMPGame.cpp @@ -476,6 +476,12 @@ void NGMPGame::launchGame(void) TheNetwork->parseUserList(this); + if (pMesh != nullptr && TheNetwork->GetConnectionManager() != nullptr) + { + // the run ahead of frame 1, decided by the packet router from the lobby round trips + pMesh->BeginStartRunAhead((int)TheNetwork->GetConnectionManager()->getPacketRouterSlot()); + } + if (TheGameLogic->isInGame()) { TheGameLogic->clearGameData(); } diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp index 477e0d7b5e8..60a8819ee98 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NetworkMesh.cpp @@ -4,6 +4,7 @@ #include "GameNetwork/GeneralsOnline/NGMP_interfaces.h" #include +#include #include #include #include "GameNetwork/NetworkDefs.h" @@ -1348,6 +1349,661 @@ void NetworkMesh::OnKeyAnnouncement(int64_t connectionUserID, const uint8_t* dat } } +// Start run ahead payload after the channel byte, little endian: +// report u8 1, u64 match id, u8 count, u8 flags (bit 0: sends directly to every peer), count x (i64 user id, u16 p75 round trip ms) +// decision u8 2, u64 match id, u8 run ahead, u16 highest pair round trip ms, u8 pairs +// ready u8 3, u64 match id, u8 stage, u8 flags (bit 0: the sender sends directly to every peer, or waited long enough for that) +// go u8 4, u64 match id, u8 stage, u16 ms the receiver waits before it starts +static constexpr uint8_t START_MSG_REPORT = 1; +static constexpr uint8_t START_MSG_DECISION = 2; +static constexpr int64_t START_REPORT_WAIT_MS = 3000; // the router decides without the reports still missing after this +static constexpr size_t START_REPORT_MAX = 16; +static constexpr int START_REPORT_MAX_RTT_MS = 2000; // longer lobby round trips are clamped +static constexpr int64_t START_LOST_GRACE_MS = 2000; // a connection that is down this long belongs to a player that left +static constexpr uint8_t START_MSG_READY = 3; +static constexpr uint8_t START_MSG_GO = 4; +static constexpr int64_t SYNC_TIMEOUT_MS = 10000; // the load screen leaves without the readies still missing after this +static constexpr int64_t SYNC_DIRECT_WAIT_MS = 3000; // a client that has no direct path to every peer by now is ready anyway, the router path carries it +static constexpr int64_t SYNC_GO_WAIT_MS = 1500; // the router's go is awaited this long after every ready is in +static constexpr int SYNC_GO_MAX_DELAY_MS = 250; + +static void WriteStartBytes(std::vector& out, const void* pData, size_t len) +{ + const uint8_t* pBytes = static_cast(pData); + out.insert(out.end(), pBytes, pBytes + len); +} + +void NetworkMesh::SendStartMessage(int64_t userID, const std::vector& payload) +{ + PlayerConnection* pConnection = GetConnectionForUser(userID); + if (pConnection == nullptr || pConnection->m_hSteamConnection == k_HSteamNetConnection_Invalid) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[RUNAHEAD] No connection to user %lld for the start run ahead", static_cast(userID)); + return; + } + + std::vector packet; + packet.push_back(static_cast(ENetworkChannel::NETWORK_CHANNEL_START)); + packet.insert(packet.end(), payload.begin(), payload.end()); + const EResult r = SteamNetworkingSockets()->SendMessageToConnection(pConnection->m_hSteamConnection, packet.data(), static_cast(packet.size()), k_nSteamNetworkingSend_ReliableNoNagle, nullptr); + if (r != k_EResultOK) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[RUNAHEAD] Could not send the start run ahead message to user %lld, error %d", static_cast(userID), static_cast(r)); + } +} + +void NetworkMesh::BeginStartRunAhead(int routerSlot) +{ + std::lock_guard lock(m_mapConnectionsMutex); + + NGMP_OnlineServices_AuthInterface* pAuthInterface = NGMP_OnlineServicesManager::GetInterface(); + NGMP_OnlineServices_LobbyInterface* pLobbyInterface = NGMP_OnlineServicesManager::GetInterface(); + if (TheNetwork == nullptr || TheNGMPGame == nullptr || pAuthInterface == nullptr || pLobbyInterface == nullptr) + { + return; + } + + const int64_t localUserID = pAuthInterface->GetUserID(); + m_startLocalUserID = localUserID; + m_startMatchID = pLobbyInterface->GetCurrentMatchID(); + m_startActive = false; + m_startDecided = false; + m_startPlayers.clear(); + m_startRouterUserID = -1; + m_startLostSinceMs.clear(); + for (Int slotNum = 0; slotNum < MAX_SLOTS; ++slotNum) + { + const NGMPGameSlot* pSlot = (const NGMPGameSlot*)TheNGMPGame->getConstSlot(slotNum); + if (pSlot == nullptr || !pSlot->isHuman()) + { + continue; + } + if (slotNum == routerSlot) + { + m_startRouterUserID = pSlot->m_userID; + } + if (pSlot->m_userID != localUserID) + { + m_startPlayers.push_back(pSlot->m_userID); + } + } + + // reports of another match are stale, reports from a user who is no start player are dropped + for (auto it = m_startReports.begin(); it != m_startReports.end(); ) + { + if (it->second.matchID != m_startMatchID || !IsStartPlayer(it->first)) + { + it = m_startReports.erase(it); + } + else + { + SanitizeStartReport(it->first, it->second); + ++it; + } + } + + for (SyncStage& st : m_sync) + { + for (auto it = st.ready.begin(); it != st.ready.end(); ) + { + it = (it->second.matchID != m_startMatchID) ? st.ready.erase(it) : std::next(it); + } + st.hasGo = st.hasGo && st.goMatchID == m_startMatchID && st.goFromUserID == m_startRouterUserID; + st.active = false; + } + + // no remote player or anticheat transport: retail start + if (m_startPlayers.empty() || m_startRouterUserID < 0 || AnticheatPlugInterface::DoesACPluginProvideSecureGameTransport()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[RUNAHEAD] start R=%d, no start run ahead exchange (players=%d, router=%lld)", (int)TheNetwork->getRunAhead(), (int)m_startPlayers.size(), static_cast(m_startRouterUserID)); + return; + } + + m_startIsRouter = (m_startRouterUserID == localUserID); + + TheNetwork->setStartRunAheadRequired(TRUE); + m_startActive = true; + m_startBeginMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + + // our round trip to every other player, the router keeps its own as a report + StartRunAheadReport own; + own.matchID = m_startMatchID; + own.directSend = TheNetwork->GetConnectionManager() != nullptr && TheNetwork->GetConnectionManager()->isDirectSendActive(true); + for (int64_t userID : m_startPlayers) + { + PlayerConnection* pConnection = GetConnectionForUser(userID); + const int rtt = (pConnection != nullptr) ? pConnection->GetSeedLatency() : -1; + if (rtt >= 0) + { + own.rtts.push_back(std::make_pair(userID, rtt)); + } + } + + if (m_startIsRouter) + { + m_startReports[localUserID] = own; + TickStartRunAhead(m_startBeginMs); + return; + } + + std::vector payload; + payload.push_back(START_MSG_REPORT); + WriteStartBytes(payload, &m_startMatchID, sizeof(m_startMatchID)); + const uint8_t count = static_cast((std::min)(own.rtts.size(), START_REPORT_MAX)); + payload.push_back(count); + payload.push_back(own.directSend ? 1 : 0); + for (uint8_t i = 0; i < count; ++i) + { + const int64_t userID = own.rtts[i].first; + const uint16_t rtt = static_cast((std::min)(own.rtts[i].second, 0xFFFE)); + WriteStartBytes(payload, &userID, sizeof(userID)); + WriteStartBytes(payload, &rtt, sizeof(rtt)); + } + SendStartMessage(m_startRouterUserID, payload); + + // the router may have decided before this client began + ApplyStartDecision(); +} + +void NetworkMesh::OnStartRunAheadMessage(int64_t connectionUserID, const uint8_t* data, size_t len) +{ + std::lock_guard lock(m_mapConnectionsMutex); + + if (data == nullptr || len < 1 + sizeof(uint64_t)) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[RUNAHEAD] Ignoring a malformed start run ahead message from user %lld", static_cast(connectionUserID)); + return; + } + + uint64_t matchID = 0; + memcpy(&matchID, data + 1, sizeof(matchID)); + size_t offset = 1 + sizeof(matchID); + + if (data[0] == START_MSG_REPORT) + { + if (len < offset + 1) + { + return; + } + const uint8_t count = data[offset++]; + if (len < offset + 1) + { + return; + } + const uint8_t flags = data[offset++]; + if (len < offset + count * (sizeof(int64_t) + sizeof(uint16_t))) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[RUNAHEAD] Ignoring a truncated round trip report from user %lld", static_cast(connectionUserID)); + return; + } + + StartRunAheadReport report; + report.matchID = matchID; + report.directSend = (flags & 1) != 0; + for (uint8_t i = 0; i < count; ++i) + { + int64_t userID = 0; + uint16_t rtt = 0; + memcpy(&userID, data + offset, sizeof(userID)); + offset += sizeof(userID); + memcpy(&rtt, data + offset, sizeof(rtt)); + offset += sizeof(rtt); + report.rtts.push_back(std::make_pair(userID, static_cast(rtt))); + } + if (m_startActive && matchID == m_startMatchID) + { + // the start players are known: only they report, and only about start players + if (!IsStartPlayer(connectionUserID)) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] Ignoring a start round trip report from user %lld, who is not a start player", static_cast(connectionUserID)); + return; + } + SanitizeStartReport(connectionUserID, report); + } + m_startReports[connectionUserID] = report; + + const int64_t nowMs = std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + TickStartRunAhead(nowMs); + } + else if (data[0] == START_MSG_DECISION) + { + if (len < offset + 1 + sizeof(uint16_t) + 1) + { + return; + } + if (m_startActive && connectionUserID != m_startRouterUserID) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] Ignoring a start run ahead decision from user %lld, who is not the router", static_cast(connectionUserID)); + return; + } + uint16_t maxRtt = 0; + StartDecision decision; + decision.matchID = matchID; + decision.runAhead = data[offset]; + memcpy(&maxRtt, data + offset + 1, sizeof(maxRtt)); + decision.maxRtt = maxRtt; + decision.pairs = data[offset + 1 + sizeof(uint16_t)]; + m_startDecisions[connectionUserID] = decision; + + ApplyStartDecision(); + } + else if (data[0] == START_MSG_READY) + { + if (len < offset + 2 || data[offset] >= SYNC_STAGES) + { + return; + } + const int stage = data[offset]; + if (m_startActive && matchID == m_startMatchID && !IsStartPlayer(connectionUserID)) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] Ignoring a start ready from user %lld, who is not a start player", static_cast(connectionUserID)); + return; + } + StartReady ready; + ready.matchID = matchID; + ready.direct = (data[offset + 1] & 1) != 0; + m_sync[stage].ready[connectionUserID] = ready; + + int total = 0; + const int missing = CountSyncMissing(stage, NowSteadyMs(), total); + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SYNC] stage %d ready from user %lld (direct=%d), %d of %d heard", stage, static_cast(connectionUserID), (int)ready.direct, total - missing, total); + } + else if (data[0] == START_MSG_GO) + { + if (len < offset + 1 + sizeof(uint16_t) || data[offset] >= SYNC_STAGES) + { + return; + } + if (m_startActive && connectionUserID != m_startRouterUserID) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SECURITY] Ignoring a start go from user %lld, who is not the router", static_cast(connectionUserID)); + return; + } + const int stage = data[offset]; + uint16_t delayMs = 0; + memcpy(&delayMs, data + offset + 1, sizeof(delayMs)); + SyncStage& st = m_sync[stage]; + st.goMatchID = matchID; + st.goDelayMs = (std::min)(static_cast(delayMs), SYNC_GO_MAX_DELAY_MS); + st.goReceivedMs = NowSteadyMs(); + st.goFromUserID = connectionUserID; + st.hasGo = true; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SYNC] stage %d go from user %lld, starting in %d ms", stage, static_cast(connectionUserID), st.goDelayMs); + } +} + +void NetworkMesh::ApplyStartDecision() +{ + if (!m_startActive || m_startIsRouter || m_startDecided || TheNetwork == nullptr) + { + return; + } + // decisions of other users are kept until the router is known, only the router's counts + auto it = m_startDecisions.find(m_startRouterUserID); + if (it == m_startDecisions.end()) + { + return; + } + if (it->second.matchID != m_startMatchID) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[RUNAHEAD] Ignoring the start R of match %llu, this is match %llu", static_cast(it->second.matchID), static_cast(m_startMatchID)); + return; + } + + m_startDecided = true; + TheNetwork->applyStartRunAhead(it->second.runAhead, it->second.maxRtt, it->second.pairs); +} + +bool NetworkMesh::StartReportsComplete() const +{ + for (int64_t userID : m_startPlayers) + { + auto it = m_startReports.find(userID); + if (it == m_startReports.end() || it->second.matchID != m_startMatchID) + { + return false; + } + } + return true; +} + +bool NetworkMesh::IsStartPlayer(int64_t userID) const +{ + return std::find(m_startPlayers.begin(), m_startPlayers.end(), userID) != m_startPlayers.end(); +} + +void NetworkMesh::SanitizeStartReport(int64_t reporter, StartRunAheadReport& report) const +{ + // an entry names the user the reporter measured: a start player or the local user, never the reporter itself + for (auto it = report.rtts.begin(); it != report.rtts.end(); ) + { + if (it->first == reporter || (!IsStartPlayer(it->first) && it->first != m_startLocalUserID)) + { + it = report.rtts.erase(it); + } + else + { + it->second = (std::min)(it->second, START_REPORT_MAX_RTT_MS); + ++it; + } + } +} + +bool NetworkMesh::IsStartPlayerLost(int64_t userID, int64_t nowMs) +{ + std::lock_guard lock(m_mapConnectionsMutex); + const PlayerConnection* pConnection = GetConnectionForUser(userID); + if (pConnection != nullptr && pConnection->IsValid()) + { + m_startLostSinceMs.erase(userID); + return false; + } + auto it = m_startLostSinceMs.emplace(userID, nowMs).first; + return nowMs - it->second >= START_LOST_GRACE_MS; +} + +void NetworkMesh::PruneLostStartPlayers(int64_t nowMs) +{ + for (auto it = m_startPlayers.begin(); it != m_startPlayers.end(); ) + { + if (IsStartPlayerLost(*it, nowMs)) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[RUNAHEAD] user %lld left before the start run ahead, not waiting for its report", static_cast(*it)); + m_startReports.erase(*it); + it = m_startPlayers.erase(it); + } + else + { + ++it; + } + } +} + +void NetworkMesh::FallBackToRetailStart() +{ + m_startDecided = true; + if (TheNetwork == nullptr) + { + return; + } + NetworkLog(ELogVerbosity::LOG_RELEASE, "[RUNAHEAD] the router (user %lld) left before it decided, start R=%d as in retail", static_cast(m_startRouterUserID), (int)TheNetwork->getRunAhead()); + TheNetwork->applyStartRunAhead((Int)TheNetwork->getRunAhead(), 0, 0); +} + +void NetworkMesh::TickStartRunAhead(int64_t nowMs) +{ + if (!m_startActive || m_startDecided) + { + return; + } + + if (!m_startIsRouter) + { + if (IsStartPlayerLost(m_startRouterUserID, nowMs)) + { + FallBackToRetailStart(); + } + return; + } + + PruneLostStartPlayers(nowMs); + if (!StartReportsComplete() && nowMs - m_startBeginMs < START_REPORT_WAIT_MS) + { + return; + } + + DecideStartRunAhead(); +} + +void NetworkMesh::DecideStartRunAhead() +{ + m_startDecided = true; + if (TheNetwork == nullptr) + { + return; + } + + // round trips count only if every player reported, otherwise everybody starts at the retail value + const bool allReported = StartReportsComplete(); + + // a pair is the higher round trip of the two its players measured to each other + std::map, int> pairRtts; + int reports = 0; + + // a command travels one way only if every player sends directly right now: the router by its own state, the others by what they reported + ConnectionManager* pConnectionManager = TheNetwork->GetConnectionManager(); + bool oneWay = pConnectionManager != nullptr && pConnectionManager->usesOneWayRunAhead(true); + for (auto& kvReport : m_startReports) + { + // a player that left after it reported does not count + const bool reporterStays = kvReport.first == m_startLocalUserID || IsStartPlayer(kvReport.first); + if (!allReported || kvReport.second.matchID != m_startMatchID || !reporterStays) + { + continue; + } + ++reports; + if (kvReport.first != m_startLocalUserID) + { + oneWay = oneWay && kvReport.second.directSend; + } + for (const auto& entry : kvReport.second.rtts) + { + if (entry.first != m_startLocalUserID && !IsStartPlayer(entry.first)) + { + continue; + } + const int64_t a = (std::min)(kvReport.first, entry.first); + const int64_t b = (std::max)(kvReport.first, entry.first); + int& rtt = pairRtts[std::make_pair(a, b)]; + rtt = (std::max)(rtt, entry.second); + } + } + + int maxRtt = 0; + for (const auto& kvPair : pairRtts) + { + maxRtt = (std::max)(maxRtt, kvPair.second); + } + + const int pairs = static_cast(pairRtts.size()); + int runAhead = static_cast(TheNetwork->getRunAhead()); + if (pairs > 0 && pConnectionManager != nullptr) + { + runAhead = pConnectionManager->computeStartRunAhead(static_cast(maxRtt), oneWay); + } + NetworkLog(ELogVerbosity::LOG_RELEASE, "[RUNAHEAD] router decides the start R=%d from %d reports of %d players (pairs=%d, max rtt=%dms, one_way=%d)%s", runAhead, reports, (int)m_startPlayers.size() + 1, pairs, maxRtt, (int)oneWay, + allReported ? "" : ", a player did not report: retail start"); + + std::vector payload; + payload.push_back(START_MSG_DECISION); + WriteStartBytes(payload, &m_startMatchID, sizeof(m_startMatchID)); + payload.push_back(static_cast(runAhead)); + const uint16_t maxRtt16 = static_cast((std::min)(maxRtt, 0xFFFE)); + WriteStartBytes(payload, &maxRtt16, sizeof(maxRtt16)); + payload.push_back(static_cast((std::min)(pairs, 255))); + for (int64_t userID : m_startPlayers) + { + SendStartMessage(userID, payload); + } + + TheNetwork->applyStartRunAhead(runAhead, maxRtt, pairs); +} + +void NetworkMesh::SendSyncReady(int stage, int64_t nowMs, bool direct) +{ + std::vector payload; + payload.push_back(START_MSG_READY); + WriteStartBytes(payload, &m_startMatchID, sizeof(m_startMatchID)); + payload.push_back(static_cast(stage)); + payload.push_back(direct ? 1 : 0); + for (int64_t userID : m_startPlayers) + { + if (!IsStartPlayerLost(userID, nowMs)) + { + SendStartMessage(userID, payload); + } + } + m_sync[stage].readySent = true; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SYNC] stage %d ready sent after %lld ms (direct=%d)", stage, static_cast(nowMs - m_sync[stage].beginMs), (int)direct); +} + +int NetworkMesh::CountSyncMissing(int stage, int64_t nowMs, int& total) +{ + int missing = 0; + total = 0; + for (int64_t userID : m_startPlayers) + { + if (IsStartPlayerLost(userID, nowMs)) + { + continue; + } + ++total; + auto it = m_sync[stage].ready.find(userID); + if (it == m_sync[stage].ready.end() || it->second.matchID != m_startMatchID) + { + ++missing; + } + } + return missing; +} + +void NetworkMesh::FinishSyncedStart(int stage, const char* szReason, int64_t nowMs) +{ + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SYNC] stage %d done %lld ms after the local part was done: %s", stage, static_cast(nowMs - m_sync[stage].beginMs), szReason); + m_sync[stage].active = false; +} + +bool NetworkMesh::BeginSyncedStart(int stage) +{ + std::lock_guard lock(m_mapConnectionsMutex); + + if (stage < 0 || stage >= SYNC_STAGES) + { + return false; + } + SyncStage& st = m_sync[stage]; + st.active = m_startActive && !m_startPlayers.empty(); + st.readySent = false; + st.allInMs = 0; + st.leaveAtMs = 0; + st.beginMs = NowSteadyMs(); + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SYNC] stage %d local part done, waiting for %d players (router=%d)", stage, st.active ? (int)m_startPlayers.size() : 0, (int)m_startIsRouter); + return st.active; +} + +bool NetworkMesh::SyncedStartDone(int stage) +{ + std::lock_guard lock(m_mapConnectionsMutex); + + if (stage < 0 || stage >= SYNC_STAGES || !m_sync[stage].active) + { + return true; + } + SyncStage& st = m_sync[stage]; + const int64_t nowMs = NowSteadyMs(); + + if (!st.readySent) + { + // on the load screen: ready once the direct path to every peer is up, or if direct sending is off or does not come up in time + bool directUp = true; + ConnectionManager* pConnectionManager = TheNetwork != nullptr ? TheNetwork->GetConnectionManager() : nullptr; + if (stage == 0 && pConnectionManager != nullptr && pConnectionManager->getNetworkSettings().send.direct) + { + directUp = pConnectionManager->isDirectSendActive(true); + } + if (directUp || nowMs - st.beginMs >= SYNC_DIRECT_WAIT_MS) + { + SendSyncReady(stage, nowMs, directUp); + } + } + + if (st.leaveAtMs != 0) + { + if (nowMs < st.leaveAtMs) + { + return false; + } + FinishSyncedStart(stage, "common start time", nowMs); + return true; + } + + if (nowMs - st.beginMs >= SYNC_TIMEOUT_MS) + { + int total = 0; + const int missing = CountSyncMissing(stage, nowMs, total); + FinishSyncedStart(stage, missing > 0 ? "timeout, a player is not ready" : "timeout", nowMs); + return true; + } + + // the router's go decides, even if a ready of ours is still on the way + if (!m_startIsRouter && st.hasGo && st.goMatchID == m_startMatchID) + { + st.leaveAtMs = st.goReceivedMs + st.goDelayMs; + return false; + } + + if (!st.readySent) + { + return false; + } + int total = 0; + const int missing = CountSyncMissing(stage, nowMs, total); + if (missing > 0) + { + st.allInMs = 0; + return false; + } + if (st.allInMs == 0) + { + st.allInMs = nowMs; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SYNC] stage %d every player is ready after %lld ms", stage, static_cast(nowMs - st.beginMs)); + } + + if (m_startIsRouter) + { + // every client starts when the farthest go has arrived: the wait is that one way time minus its own + int maxOneWay = 0; + for (int64_t userID : m_startPlayers) + { + const PlayerConnection* pConnection = GetConnectionForUser(userID); + const int rtt = pConnection != nullptr ? pConnection->GetSeedLatency() : -1; + maxOneWay = (std::max)(maxOneWay, (std::min)((std::max)(rtt, 0) / 2, SYNC_GO_MAX_DELAY_MS)); + } + for (int64_t userID : m_startPlayers) + { + if (IsStartPlayerLost(userID, nowMs)) + { + continue; + } + const PlayerConnection* pConnection = GetConnectionForUser(userID); + const int rtt = pConnection != nullptr ? pConnection->GetSeedLatency() : -1; + const uint16_t delayMs = static_cast((std::max)(maxOneWay - (std::min)((std::max)(rtt, 0) / 2, SYNC_GO_MAX_DELAY_MS), 0)); + std::vector payload; + payload.push_back(START_MSG_GO); + WriteStartBytes(payload, &m_startMatchID, sizeof(m_startMatchID)); + payload.push_back(static_cast(stage)); + WriteStartBytes(payload, &delayMs, sizeof(delayMs)); + SendStartMessage(userID, payload); + } + st.leaveAtMs = nowMs + maxOneWay; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[SYNC] stage %d go sent to %d players, starting in %d ms", stage, total, maxOneWay); + if (maxOneWay == 0) + { + FinishSyncedStart(stage, "common start time", nowMs); + return true; + } + return false; + } + + // no go from a router that left or does not answer: everybody has its ready, start as the others do + if (IsStartPlayerLost(m_startRouterUserID, nowMs) || nowMs - st.allInMs >= SYNC_GO_WAIT_MS) + { + FinishSyncedStart(stage, "every ready is in, no go from the router", nowMs); + return true; + } + return false; +} + void NetworkMesh::AwaitTurnCredentials() { std::lock_guard lock(m_mapConnectionsMutex); @@ -1513,6 +2169,7 @@ void NetworkMesh::Tick() } TickLobbyKeys(nowMs); + TickStartRunAhead(nowMs); // the game transport isn't created until the game begins, but we want to transfer AC packets in the lobby first, so consider this a liteupdate if (TheNGMPGame != nullptr && !TheNGMPGame->isGameInProgress()) @@ -1596,6 +2253,16 @@ void PlayerConnection::LiteUpdateForAC() msg->Release(); continue; } + else if (netChannel == ENetworkChannel::NETWORK_CHANNEL_START) + { + NetworkMesh* pMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + if (pMesh != nullptr) + { + pMesh->OnStartRunAheadMessage(m_userID, vecData.data() + sizeof(ENetworkChannel), numBytes - sizeof(ENetworkChannel)); + } + msg->Release(); + continue; + } } else if (numBytes != -1 && numBytes < sizeof(ENetworkChannel)) { @@ -2197,8 +2864,8 @@ float PlayerConnection::GetSustainedLossPercent(int windowMs) int PlayerConnection::GetJitter() { - // smoothed round trip variation of the probe over its stats window - if (m_probeStats.getSampleCount() < 10) + // RFC 3550 style: the mean deviation of consecutive probe round trips, smoothed with a gain of 1/16 + if (m_probeStats.getSampleCount() < k_seedMinSamples) { return -1; } @@ -2206,6 +2873,17 @@ int PlayerConnection::GetJitter() return static_cast(m_probeStats.getJitter()); } +int PlayerConnection::GetMatchRatingMinSamples() const +{ + int intervalMs = 1000; + if (NGMP_OnlineServicesManager::GetInstance() != nullptr) + { + intervalMs = (std::max)(1, NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig().connection.probe_match_interval_ms); + } + // the window holds about W / interval probes, 10 at 1 s: the probe value stays until more than half of them are lost + return (std::min)(k_seedMinSamples, (std::max)(3, NETWORK_STATS_WINDOW_MS / (2 * intervalMs))); +} + int PlayerConnection::ComputeConnectionRating() { // the overlay and the lobby ask every frame, the percentile sorts inside do not change that fast @@ -2232,9 +2910,6 @@ int PlayerConnection::UpdateConnectionRating() { return -1; } - const ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - const int slackPercent = serviceConf.input_delay.slack_percent; - int sendIntervalMs = serviceConf.send.frames_per_packet * 1000 / GENERALS_ONLINE_HIGH_FPS_LIMIT; // the command connections are indexed by game slot, find the slot of this user int slot = -1; @@ -2262,17 +2937,26 @@ int PlayerConnection::UpdateConnectionRating() } const float sustainedLoss = GetSustainedLossPercent(feedbackWindowMs); + ConnectionManager *commandManager = TheNetwork != nullptr ? TheNetwork->GetConnectionManager() : nullptr; + const ServiceConfig &lobbyConfig = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); + const NetworkSettings::Send &ratingSend = commandManager != nullptr + ? commandManager->getNetworkSettings().send : lobbyConfig.send; + const NetworkSettings::InputDelay &ratingInputDelay = commandManager != nullptr + ? commandManager->getNetworkSettings().input_delay : lobbyConfig.input_delay; + // the state of this link, a pair that fell back to the packet router is rated with the longer path + const Bool oneWay = commandManager != nullptr ? commandManager->usesOneWayRunAheadForUser(m_userID) + : lobbyConfig.input_delay.controller && lobbyConfig.send.direct && m_State == EConnectionState::CONNECTED_DIRECT; + if (inMatch && TheNetwork->getOverlayInfo(slot, info)) { - if (info.frameGroupingMs > 0) - { - sendIntervalMs = info.frameGroupingMs; - } - inputs = ConnectionRating::InputsFromMatch(*pCmd, info, slackPercent, sendIntervalMs, feedbackLoss); + // probe p75, else the smoothed command round trip + const int seedLatency = GetSeedLatency(GetMatchRatingMinSamples()); + const Real roundTripMs = (seedLatency >= 0) ? (Real)seedLatency : pCmd->getLatency(); + inputs = ConnectionRating::InputsFromMatch(*pCmd, info, feedbackLoss, ConnectionManager::estimateRunAheadMs(roundTripMs, ratingSend, ratingInputDelay, oneWay)); } - else if (m_probeStats.getSampleCount() >= 10) + else if (GetSeedLatency() >= 0) { - inputs = ConnectionRating::InputsFromProbe(m_probeStats, slackPercent, sendIntervalMs, feedbackLoss); + inputs = ConnectionRating::InputsFromProbe(m_probeStats, feedbackLoss, ConnectionManager::estimateRunAheadMs((Real)GetSeedLatency(), ratingSend, ratingInputDelay, oneWay)); } else { @@ -2288,7 +2972,7 @@ int PlayerConnection::UpdateConnectionRating() } { // silence must outlast the probe interval of the current phase, probes may be the only traffic - const ServiceConfig::Connection& pingConfig = serviceConf.connection; + const ServiceConfig::Connection& pingConfig = lobbyConfig.connection; const int pingIntervalMs = (TheNetwork != nullptr) ? pingConfig.probe_match_interval_ms : pingConfig.probe_lobby_interval_ms; inputs.silentLimitMs = (std::max)(inputs.silentLimitMs, pingIntervalMs * 5 / 2); } diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp index 8c387770c45..4765fb3ccd1 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/NextGenTransport.cpp @@ -405,6 +405,16 @@ Bool NextGenTransport::doRecv(void) msg->Release(); continue; } + else if (netChannel == ENetworkChannel::NETWORK_CHANNEL_START) + { + NetworkMesh* pStartMesh = NGMP_OnlineServicesManager::GetNetworkMesh(); + if (pStartMesh != nullptr) + { + pStartMesh->OnStartRunAheadMessage(kvPair.second.m_userID, reinterpret_cast(vecPacketDataWithoutHeader.data()), vecPacketDataWithoutHeader.size()); + } + msg->Release(); + continue; + } else if (netChannel == ENetworkChannel::NETWORK_CHANNEL_PING) { kvPair.second.HandleProbe(reinterpret_cast(vecPacketDataWithoutHeader.data()), vecPacketDataWithoutHeader.size(), PlayerConnection::GetArrivalMs(msg)); diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp index 29ae73bc5bd..680a768491f 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp @@ -10,6 +10,7 @@ #include #include "GameClient/DisplayStringManager.h" #include "GameNetwork/NetworkInterface.h" +#include "GameNetwork/ConnectionManager.h" #include "Common/MultiplayerSettings.h" #include "GameNetwork/GameSpyOverlay.h" #include "GameClient/Display.h" @@ -141,6 +142,12 @@ void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType scree { CHECK_MAIN_THREAD; + // the capture stalls every client at the same time, the run ahead controller must not read that as the network + if (TheNetwork != nullptr && TheNetwork->GetConnectionManager() != nullptr) + { + TheNetwork->GetConnectionManager()->noteScreenshotCapture(screenshotType == EScreenshotType::SCREENSHOT_TYPE_GAMEPLAY); + } + NGMP_OnlineServices_LobbyInterface* pLobbyInterface = NGMP_OnlineServicesManager::GetInterface(); if (pLobbyInterface != nullptr) { @@ -979,6 +986,11 @@ void NGMP_OnlineServicesManager::Tick() } else // we have the data we need, send away { + // the delayed upload starts long after the capture, hold the controller from here + if (screenshotEntry.screenshotType == EScreenshotType::SCREENSHOT_TYPE_GAMEPLAY && TheNetwork != nullptr && TheNetwork->GetConnectionManager() != nullptr) + { + TheNetwork->GetConnectionManager()->noteScreenshotCapture(true); + } std::map mapHeaders; mapHeaders["Content-Type"] = "image/jpeg"; NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()->SendS3PUTRequest(screenshotEntry.strSignedURI.c_str(), EIPProtocolVersion::DONT_CARE, mapHeaders, std::move(screenshotEntry.vecBytes), [=](bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq) @@ -990,6 +1002,10 @@ void NGMP_OnlineServicesManager::Tick() } #endif NetworkLog(ELogVerbosity::LOG_RELEASE, "Screenshot upload, result: %d", statusCode); + if (screenshotEntry.screenshotType == EScreenshotType::SCREENSHOT_TYPE_GAMEPLAY && TheNetwork != nullptr && TheNetwork->GetConnectionManager() != nullptr) + { + TheNetwork->GetConnectionManager()->noteScreenshotDone(); + } }, nullptr, HTTP_UPLOAD_TIMEOUT, bUnthrottled ? 0 : uploadBytesPerSec); }