From 0a59aec1ba38a528ffabf90d7dd2ad245d76d581 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:37:40 +0200 Subject: [PATCH 01/17] perf(screenshot): Avoid redundant buffers and copies in screenshot uploads --- .../GeneralsOnline/OnlineServices_Init.h | 4 +- .../GeneralsOnline/HTTP/HTTPManager.cpp | 2 +- .../GeneralsOnline/OnlineServices_Init.cpp | 203 ++++++++---------- 3 files changed, 88 insertions(+), 121 deletions(-) diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index 7d1419d2ee1..27d5fffdfcd 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -586,10 +586,10 @@ class NGMP_OnlineServicesManager ServiceConfig& GetServiceConfig() { return m_ServiceConfig; } public: - void CacheScreenshotBytes_StartMatch(std::vector& vecData) + void CacheScreenshotBytes_StartMatch(std::vector vecData) { std::scoped_lock ssLock(m_ScreenshotMutex); - m_vecCachedScreenshotBytes_MatchStart = vecData; + m_vecCachedScreenshotBytes_MatchStart = std::move(vecData); } void CacheScreenshotBytes_EndMatch(uint64_t matchID, std::vector data); diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp index a420a04cda6..9b6d94eca4c 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp @@ -49,7 +49,7 @@ void HTTPManager::SendS3PUTRequest(const char* szURI, EIPProtocolVersion protove HTTPRequest* pRequest = PlatformCreateRequest(EHTTPVerb::HTTP_VERB_PUT, protover, szURI, inHeaders, completionCallback, progressCallback, timeoutMS); pRequest->DisableServiceAuth(); - pRequest->SetPostDataBuffer(vecBuffer); + pRequest->SetPostDataBuffer(std::move(vecBuffer)); m_vecRequestsPendingStart.push_back(pRequest); } diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp index cfe0835efd4..951c40f6dd2 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp @@ -127,7 +127,7 @@ void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType scree // certain screenshots require caching for later upload when we have a valid match id, so we store them if (screenshotType == EScreenshotType::SCREENSHOT_TYPE_LOADSCREEN) { - NGMP_OnlineServicesManager::GetInstance()->CacheScreenshotBytes_StartMatch(vecData); + NGMP_OnlineServicesManager::GetInstance()->CacheScreenshotBytes_StartMatch(std::move(vecData)); } else if (screenshotType == EScreenshotType::SCREENSHOT_TYPE_SCORESCREEN) { @@ -143,7 +143,7 @@ void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType scree newEntry.vecBytes = std::move(vecData); newEntry.strSignedURI = strURI; newEntry.screenshotType = screenshotType; - m_vecGuardedSSData.push_back(newEntry); + m_vecGuardedSSData.push_back(std::move(newEntry)); } }); } @@ -579,7 +579,6 @@ void NGMP_OnlineServicesManager::CaptureScreenshot(bool bResizeForTransmit, std: } SurfaceClass* surface = DX8Wrapper::_Get_DX8_Back_Buffer(); - LPDIRECT3DSURFACE8 surf = nullptr; SurfaceClass* surfaceCopy = nullptr; void* pBits = nullptr; IDirect3DSurface8* pDXsurf = nullptr; @@ -599,119 +598,96 @@ void NGMP_OnlineServicesManager::CaptureScreenshot(bool bResizeForTransmit, std: { DX8Wrapper::_Copy_DX8_Rects(surface->Peek_D3D_Surface(), NULL, 0, surfaceCopy->Peek_D3D_Surface(), NULL); - HRESULT hr; + // gather all our data + int pitch = 0; + pBits = surfaceCopy->Lock(&pitch); - D3DDISPLAYMODE mode; - if (SUCCEEDED(hr = DX8Wrapper::_Get_D3D_Device8()->GetDisplayMode(&mode))) + if (pBits != nullptr) { - if (SUCCEEDED(hr = DX8Wrapper::_Get_D3D_Device8()->CreateImageSurface(mode.Width, mode.Height, - D3DFMT_A8R8G8B8, &surf))) - { - if (SUCCEEDED(hr = DX8Wrapper::_Get_D3D_Device8()->GetFrontBuffer(surf))) + int width = surfaceDesc.Width; + int height = surfaceDesc.Height; + + // Copy pixel data into an owned buffer before spawning the thread so + // the surface can be safely unlocked on the main thread immediately after. + const uint8_t* pSurfaceBytes = static_cast(pBits); + std::vector pixelData(pSurfaceBytes, pSurfaceBytes + height * pitch); + + // process on thread - track the thread so we can join it during shutdown + std::thread* pNewThread = new std::thread([cbOnDataAvailable, width, height, pixelData = std::move(pixelData), pitch, bResizeForTransmit]() mutable { - // gather all our data - int pitch = 0; - pBits = surfaceCopy->Lock(&pitch); + CHECK_WORKER_THREAD; - if (pBits != nullptr) - { - int width = surfaceDesc.Width; - int height = surfaceDesc.Height; - - // Copy pixel data into an owned buffer before spawning the thread so - // the surface can be safely unlocked on the main thread immediately after. - std::vector pixelData(height * pitch); - memcpy(pixelData.data(), pBits, height * pitch); - - // process on thread - track the thread so we can join it during shutdown - std::thread* pNewThread = new std::thread([cbOnDataAvailable, width, height, pixelData = std::move(pixelData), pitch, bResizeForTransmit]() - { - CHECK_WORKER_THREAD; - - unsigned char* rgbData = new unsigned char[width * height * 3]; - - std::vector vecData; - - int finalWidth = width; - int finalHeight = height; - - for (int y = 0; y < height; ++y) { - const uint8_t* row = pixelData.data() + y * pitch; - int rowOffset = y * width * 3; - int srcOffset = 0; - for (int x = 0; x < width; ++x, srcOffset += 4) - { - int dstIndex = rowOffset + x * 3; - rgbData[dstIndex + 0] = row[srcOffset + 2]; // R - rgbData[dstIndex + 1] = row[srcOffset + 1]; // G - rgbData[dstIndex + 2] = row[srcOffset + 0]; // B - } - } - - // resize - unsigned char* pBufferToWrite = rgbData; - if (bResizeForTransmit) - { - ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); - int new_width = serviceConf.screenshot_width; - int new_height = serviceConf.screenshot_height; - int channels = 3; - unsigned char* resized = new unsigned char[new_width * new_height * channels]; - - stbir_resize_uint8(rgbData, width, height, 0, - resized, new_width, new_height, 0, - channels - ); - - // update data - finalWidth = new_width; - finalHeight = new_height; - pBufferToWrite = resized; - } - // end resize - - stbi_write_jpg_to_func([](void* context, void* data, int size) - { - std::vector* buffer = static_cast*>(context); - buffer->insert(buffer->end(), (unsigned char*)data, (unsigned char*)data + size); - }, &vecData, finalWidth, finalHeight, 3, pBufferToWrite, bResizeForTransmit ? 0 : 90); - - // cleanup - if (bResizeForTransmit) - { - delete[] pBufferToWrite; // This is 'resized' - pBufferToWrite = nullptr; - } - - delete[] rgbData; - rgbData = nullptr; - - // invoke cb - if (cbOnDataAvailable != nullptr) - { - cbOnDataAvailable(vecData); - } - } - ); + const int channels = 3; + int finalWidth = width; + int finalHeight = height; - // Store the thread so we can join it during shutdown - // SECURITY FIX: Capture manager pointer before spawning thread to avoid TOCTOU race - NGMP_OnlineServicesManager* pMgr = NGMP_OnlineServicesManager::GetInstance(); - if (pMgr != nullptr) + // Pack the BGRA rows to tight RGB in place. The write position never passes the read position. + uint8_t* pixels = pixelData.data(); + for (int y = 0; y < height; ++y) { - std::scoped_lock lock(pMgr->m_mutexScreenshotThreads); - pMgr->m_vecScreenshotThreads.push_back(pNewThread); + const uint8_t* src = pixels + y * pitch; + uint8_t* dst = pixels + y * width * channels; + for (int x = 0; x < width; ++x, src += 4, dst += channels) + { + const uint8_t blue = src[0]; + dst[0] = src[2]; // R + dst[1] = src[1]; // G + dst[2] = blue; // B + } } - else + + // resize + std::vector resized; + if (bResizeForTransmit) { - // Manager was destroyed, cannot store thread. Thread will leak but won't crash. - NetworkLog(ELogVerbosity::LOG_RELEASE, "[Screenshot] Manager destroyed before thread could be registered"); + ServiceConfig& serviceConf = NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig(); + finalWidth = serviceConf.screenshot_width; + finalHeight = serviceConf.screenshot_height; + resized.resize(finalWidth * finalHeight * channels); + + stbir_resize_uint8(pixels, width, height, 0, + resized.data(), finalWidth, finalHeight, 0, + channels + ); + + // the capture is no longer needed + std::vector().swap(pixelData); } + // end resize + + // room for a typical JPEG, so the write callback rarely has to grow it + std::vector vecData; + vecData.reserve(256 * 1024); + + stbi_write_jpg_to_func([](void* context, void* data, int size) + { + std::vector* buffer = static_cast*>(context); + buffer->insert(buffer->end(), (unsigned char*)data, (unsigned char*)data + size); + }, &vecData, finalWidth, finalHeight, channels, bResizeForTransmit ? resized.data() : pixelData.data(), bResizeForTransmit ? 0 : 90); - bSucceeded = true; + // invoke cb + if (cbOnDataAvailable != nullptr) + { + cbOnDataAvailable(std::move(vecData)); } } + ); + + // Store the thread so we can join it during shutdown + // SECURITY FIX: Capture manager pointer before spawning thread to avoid TOCTOU race + NGMP_OnlineServicesManager* pMgr = NGMP_OnlineServicesManager::GetInstance(); + if (pMgr != nullptr) + { + std::scoped_lock lock(pMgr->m_mutexScreenshotThreads); + pMgr->m_vecScreenshotThreads.push_back(pNewThread); + } + else + { + // Manager was destroyed, cannot store thread. Thread will leak but won't crash. + NetworkLog(ELogVerbosity::LOG_RELEASE, "[Screenshot] Manager destroyed before thread could be registered"); } + + bSucceeded = true; } } } @@ -719,18 +695,9 @@ void NGMP_OnlineServicesManager::CaptureScreenshot(bool bResizeForTransmit, std: // clean everything up, whether we succeeded or not - // release the image surface - if (surf != nullptr) - { - surf->Release(); - //delete surf; - surf = nullptr; - } - // unlock if (surface != nullptr) { - surface->Unlock(); surface->Release_Ref(); surface = nullptr; } @@ -954,7 +921,7 @@ void NGMP_OnlineServicesManager::Tick() // screenshot types that already have a presigned URL - for (S3ScreenshotEntry screenshotEntry : m_vecGuardedSSData) + for (S3ScreenshotEntry& screenshotEntry : m_vecGuardedSSData) { // NOTE: Screenshot types start and end of match are captures and cached in memory until the server tells us where to upload them, so we need to wait for the upload URI @@ -966,7 +933,7 @@ void NGMP_OnlineServicesManager::Tick() { std::map mapHeaders; mapHeaders["Content-Type"] = "image/jpeg"; - NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()->SendS3PUTRequest(screenshotEntry.strSignedURI.c_str(), EIPProtocolVersion::DONT_CARE, mapHeaders, screenshotEntry.vecBytes, [=](bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq) + NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()->SendS3PUTRequest(screenshotEntry.strSignedURI.c_str(), EIPProtocolVersion::DONT_CARE, mapHeaders, std::move(screenshotEntry.vecBytes), [=](bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq) { #if _DEBUG if (statusCode != 200) @@ -990,9 +957,9 @@ void NGMP_OnlineServicesManager::Tick() // queue it S3ScreenshotEntry newEntry; newEntry.screenshotType = EScreenshotType::SCREENSHOT_TYPE_LOADSCREEN; - newEntry.vecBytes = m_vecCachedScreenshotBytes_MatchStart; - newEntry.strSignedURI = m_strCachedScreenshot_MatchStart_S3URI; - m_vecGuardedSSData.push_back(newEntry); + newEntry.vecBytes = std::move(m_vecCachedScreenshotBytes_MatchStart); + newEntry.strSignedURI = std::move(m_strCachedScreenshot_MatchStart_S3URI); + m_vecGuardedSSData.push_back(std::move(newEntry)); // clear data m_vecCachedScreenshotBytes_MatchStart = std::vector(); @@ -1023,7 +990,7 @@ void NGMP_OnlineServicesManager::Tick() // do the upload std::map mapHeaders; mapHeaders["Content-Type"] = "application/octet-stream"; - NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()->SendS3PUTRequest(m_cachedReplayUpload.signedURI.c_str(), EIPProtocolVersion::DONT_CARE, mapHeaders, m_cachedReplayUpload.bytes, [=](bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq) + NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()->SendS3PUTRequest(m_cachedReplayUpload.signedURI.c_str(), EIPProtocolVersion::DONT_CARE, mapHeaders, std::move(m_cachedReplayUpload.bytes), [=](bool bSuccess, int statusCode, std::string strBody, HTTPRequest* pReq) { #if _DEBUG if (statusCode != 200) From 4e21dabadfc6e22df9d19667e2bc7f9d05e22e6d Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:32:41 +0100 Subject: [PATCH 02/17] perf(screenshot): Upload probe screenshots one slot at a time at a capped rate --- .../GeneralsOnline/HTTP/HTTPManager.h | 2 +- .../GeneralsOnline/HTTP/HTTPRequest.h | 7 +++ .../GeneralsOnline/OnlineServices_Init.h | 6 +- .../GeneralsOnline/HTTP/HTTPManager.cpp | 3 +- .../GeneralsOnline/HTTP/HTTPRequest.cpp | 5 ++ .../GeneralsOnline/OnlineServices_Init.cpp | 60 +++++++++++++++++-- 6 files changed, 74 insertions(+), 9 deletions(-) diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h index 57a4c498928..d2d5c80f753 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h @@ -54,7 +54,7 @@ class HTTPManager // NOTE: set bDisableServiceAuth for endpoints that must be called with something other than the session token (e.g. the refresh endpoint, which needs the refresh token), otherwise the session token overwrites any Authorization header passed in inHeaders void SendPOSTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, const char* szPostData, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1, bool bDisableServiceAuth = false); void SendPUTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, const char* szData, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1); - void SendS3PUTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, std::vector vecBuffer, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1); + void SendS3PUTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, std::vector vecBuffer, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1, int maxSendBytesPerSec = 0); void SendDELETERequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, const char* szData, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1); void Shutdown(); diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h index f5b901aade5..7dbf977a136 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h @@ -36,6 +36,11 @@ class HTTPRequest m_bAppendAuthIfPresent = false; } + void SetMaxSendSpeed(int bytesPerSec) + { + m_maxSendBytesPerSec = bytesPerSec; + } + void OnResponsePartialWrite(std::uint8_t* pBuffer, size_t numBytes); bool HasStarted() const { return m_bIsStarted; } @@ -77,6 +82,8 @@ class HTTPRequest bool m_bAppendAuthIfPresent = true; + int m_maxSendBytesPerSec = 0; // 0 = unlimited + EIPProtocolVersion m_protover; int m_timeoutMS = 5000; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index 27d5fffdfcd..2096e7b3623 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -4,6 +4,7 @@ #include #include +#include class HTTPManager; @@ -28,6 +29,7 @@ struct S3ScreenshotEntry std::vector vecBytes; std::string strSignedURI; EScreenshotType screenshotType; + std::chrono::steady_clock::time_point notBefore; // upload is held back until this time bool operator==(const S3ScreenshotEntry& other) const { @@ -341,6 +343,8 @@ struct ServiceConfig int screenshot_width = 557; int screenshot_height = 333; + int screenshot_upload_bytes_per_sec = 131072; // 128 KB/s + int screenshot_slot_offset_ms = 2000; // ICE client used for P2P NAT traversal: 0 = library default, 1 = native, 2 = WebRTC. int ice_implementation = 2; @@ -351,7 +355,7 @@ struct ServiceConfig ra_slack_percent, frame_grouping_frames, enable_host_migration, network_do_immediate_flush_per_frame, network_send_flags, network_latency_logic_model, use_default_config, ra_slack_override_percent_in_default, do_probes, do_replay_upload, network_mesh_histogram_duration, ibra_ra_tweaks, ibra_minslack_default, ibra_maxslack_default, ibra_minslack_greaterthan300ms, ibra_maxslack_greaterthan300ms, ibra_minslack_greaterthan200ms, ibra_maxslack_greaterthan200ms, - screenshot_width, screenshot_height, ice_implementation, use_new_networking) + screenshot_width, screenshot_height, screenshot_upload_bytes_per_sec, screenshot_slot_offset_ms, ice_implementation, use_new_networking) }; class NGMP_OnlineServicesManager diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp index 9b6d94eca4c..e476ced3bb7 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp @@ -43,12 +43,13 @@ void HTTPManager::SendPUTRequest(const char* szURI, EIPProtocolVersion protover, } -void HTTPManager::SendS3PUTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, std::vector vecBuffer, std::function completionCallback, std::function progressCallback /*= nullptr*/, int timeoutMS /*= -1*/) +void HTTPManager::SendS3PUTRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, std::vector vecBuffer, std::function completionCallback, std::function progressCallback /*= nullptr*/, int timeoutMS /*= -1*/, int maxSendBytesPerSec /*= 0*/) { CHECK_MAIN_THREAD; HTTPRequest* pRequest = PlatformCreateRequest(EHTTPVerb::HTTP_VERB_PUT, protover, szURI, inHeaders, completionCallback, progressCallback, timeoutMS); pRequest->DisableServiceAuth(); + pRequest->SetMaxSendSpeed(maxSendBytesPerSec); pRequest->SetPostDataBuffer(std::move(vecBuffer)); m_vecRequestsPendingStart.push_back(pRequest); diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp index 02786d7f3d9..3c77e8f4039 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp @@ -298,6 +298,11 @@ void HTTPRequest::PlatformStartRequest() } } + if (m_maxSendBytesPerSec > 0) + { + curl_easy_setopt(m_pCURL, CURLOPT_MAX_SEND_SPEED_LARGE, (curl_off_t)m_maxSendBytesPerSec); + } + // needed for PUT etc if (m_httpVerb == EHTTPVerb::HTTP_VERB_PUT) { diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp index 951c40f6dd2..b2fa378fdb1 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp @@ -23,7 +23,10 @@ #include "GameNetwork/GeneralsOnline/Vendor/stb_image/stb_image_write.h" #include "GameNetwork/GeneralsOnline/Vendor/stb_image/stb_image_resize.h" #include "GameClient/GameText.h" +#include "GameLogic/GameLogic.h" #include +#include +#include extern "C" { @@ -99,6 +102,34 @@ void NGMP_OnlineServicesManager::GetAndParseServiceConfig(std::function(); + NGMP_OnlineServices_LobbyInterface* pLobbyInterface = NGMP_OnlineServicesManager::GetInterface(); + if (pAuthInterface != nullptr && pLobbyInterface != nullptr) + { + const int64_t localUserID = pAuthInterface->GetUserID(); + for (const LobbyMemberEntry& lobbyMember : pLobbyInterface->GetMembersListForCurrentRoom()) + { + if (lobbyMember.user_id == localUserID) + { + slotIndex = lobbyMember.m_SlotIndex < MAX_SLOTS ? lobbyMember.m_SlotIndex : 0; + break; + } + } + } + + const int slotOffsetMS = std::clamp(NGMP_OnlineServicesManager::GetInstance()->GetServiceConfig().screenshot_slot_offset_ms, 0, 10000); + + static thread_local std::mt19937 rng{ std::random_device{}() }; + const int jitterMS = std::uniform_int_distribution(0, 500)(rng); + + return std::chrono::steady_clock::now() + std::chrono::milliseconds(slotIndex * slotOffsetMS + jitterMS); +} + void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType screenshotType, std::string strURI) { NGMP_OnlineServicesManager* pOnlineServicesMgr = NGMP_OnlineServicesManager::GetInstance(); @@ -114,7 +145,8 @@ void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType scree if (pLobbyInterface != nullptr) { const uint64_t matchID = pLobbyInterface->GetCurrentMatchID(); - NGMP_OnlineServicesManager::GetInstance()->CaptureScreenshot(true, [strURI = std::move(strURI), screenshotType, matchID](std::vector vecData) + const std::chrono::steady_clock::time_point uploadTime = GetScreenshotUploadTime(); + NGMP_OnlineServicesManager::GetInstance()->CaptureScreenshot(true, [strURI = std::move(strURI), screenshotType, matchID, uploadTime](std::vector vecData) { CHECK_WORKER_THREAD; @@ -143,6 +175,7 @@ void NGMP_OnlineServicesManager::CaptureScreenshotForProbe(EScreenshotType scree newEntry.vecBytes = std::move(vecData); newEntry.strSignedURI = strURI; newEntry.screenshotType = screenshotType; + newEntry.notBefore = uploadTime; m_vecGuardedSSData.push_back(std::move(newEntry)); } }); @@ -920,13 +953,28 @@ void NGMP_OnlineServicesManager::Tick() std::scoped_lock ssLock(m_ScreenshotMutex); + const int uploadBytesPerSec = std::clamp(m_ServiceConfig.screenshot_upload_bytes_per_sec, 16384, 10485760); + const std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now(); + const bool bMatchOver = TheGameLogic == nullptr || !TheGameLogic->isInMultiplayerGame(); // uploads no longer compete with game traffic + // screenshot types that already have a presigned URL - for (S3ScreenshotEntry& screenshotEntry : m_vecGuardedSSData) + for (auto it = m_vecGuardedSSData.begin(); it != m_vecGuardedSSData.end();) { + S3ScreenshotEntry& screenshotEntry = *it; + const bool bUnthrottled = bMatchOver || screenshotEntry.screenshotType == EScreenshotType::SCREENSHOT_TYPE_SCORESCREEN; + + // not this client's turn to upload yet + if (!bUnthrottled && screenshotEntry.notBefore > now) + { + ++it; + continue; + } + // NOTE: Screenshot types start and end of match are captures and cached in memory until the server tells us where to upload them, so we need to wait for the upload URI if ((screenshotEntry.screenshotType == EScreenshotType::SCREENSHOT_TYPE_LOADSCREEN || screenshotEntry.screenshotType == EScreenshotType::SCREENSHOT_TYPE_SCORESCREEN) && screenshotEntry.strSignedURI.empty()) { + it = m_vecGuardedSSData.erase(it); continue; } else // we have the data we need, send away @@ -942,12 +990,11 @@ void NGMP_OnlineServicesManager::Tick() } #endif NetworkLog(ELogVerbosity::LOG_RELEASE, "Screenshot upload, result: %d", statusCode); - }, nullptr, HTTP_UPLOAD_TIMEOUT); + }, nullptr, HTTP_UPLOAD_TIMEOUT, bUnthrottled ? 0 : uploadBytesPerSec); } - - } - m_vecGuardedSSData.clear(); + it = m_vecGuardedSSData.erase(it); + } // screenshots and replays that are cached and awaiting S3 url, and now have said URL if (!m_vecCachedScreenshotBytes_MatchStart.empty()) // we have data waiting @@ -959,6 +1006,7 @@ void NGMP_OnlineServicesManager::Tick() newEntry.screenshotType = EScreenshotType::SCREENSHOT_TYPE_LOADSCREEN; newEntry.vecBytes = std::move(m_vecCachedScreenshotBytes_MatchStart); newEntry.strSignedURI = std::move(m_strCachedScreenshot_MatchStart_S3URI); + newEntry.notBefore = GetScreenshotUploadTime(); m_vecGuardedSSData.push_back(std::move(newEntry)); // clear data From 077a6147819aa36fa09ec4cab87230e25a274762 Mon Sep 17 00:00:00 2001 From: xezon <4720891+xezon@users.noreply.github.com> Date: Sun, 26 Jul 2026 17:19:44 +0200 Subject: [PATCH 03/17] refactor(filesystem): Remove obsolete allocation in Win32BIGFileSystem::openArchiveFile() (#3376) --- .../StdDevice/Common/StdBIGFileSystem.cpp | 20 ++++++++---------- .../Win32Device/Common/Win32BIGFileSystem.cpp | 21 +++++++------------ 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp b/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp index ffd0150f4c5..0fb035e0c3e 100644 --- a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp +++ b/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp @@ -128,7 +128,8 @@ ArchiveFile * StdBIGFileSystem::openArchiveFile(const Char *filename) { // seek to the beginning of the directory listing. fp->seek(0x10, File::START); // read in each directory listing. - ArchivedFileInfo *fileInfo = NEW ArchivedFileInfo; + ArchivedFileInfo fileInfo; + ArchiveFile *archiveFile = NEW StdBIGFile(filename, AsciiString::TheEmptyString); for (Int i = 0; i < numLittleFiles; ++i) { Int filesize = 0; @@ -139,9 +140,9 @@ ArchiveFile * StdBIGFileSystem::openArchiveFile(const Char *filename) { filesize = betoh(filesize); fileOffset = betoh(fileOffset); - fileInfo->m_archiveFilename = archiveFileName; - fileInfo->m_offset = fileOffset; - fileInfo->m_size = filesize; + fileInfo.m_archiveFilename = archiveFileName; + fileInfo.m_offset = fileOffset; + fileInfo.m_size = filesize; // read in the path name of the file. Int pathIndex = -1; @@ -155,8 +156,8 @@ ArchiveFile * StdBIGFileSystem::openArchiveFile(const Char *filename) { --filenameIndex; } - fileInfo->m_filename = (char *)(buffer + filenameIndex + 1); - fileInfo->m_filename.toLower(); + fileInfo.m_filename = (char *)(buffer + filenameIndex + 1); + fileInfo.m_filename.toLower(); buffer[filenameIndex + 1] = 0; AsciiString path; @@ -164,17 +165,14 @@ ArchiveFile * StdBIGFileSystem::openArchiveFile(const Char *filename) { AsciiString debugpath; debugpath = path; - debugpath.concat(fileInfo->m_filename); + debugpath.concat(fileInfo.m_filename); // DEBUG_LOG(("StdBIGFileSystem::openArchiveFile - adding file %s to archive file %s, file number %d", debugpath.str(), fileInfo->m_archiveFilename.str(), i)); - archiveFile->addFile(path, fileInfo); + archiveFile->addFile(path, &fileInfo); } archiveFile->attachFile(fp); - delete fileInfo; - fileInfo = nullptr; - // leave fp open as the archive file will be using it. return archiveFile; diff --git a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp b/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp index b824dcaed57..004e393ec02 100644 --- a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp +++ b/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp @@ -137,8 +137,7 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { // seek to the beginning of the directory listing. fp->seek(0x10, File::START); // read in each directory listing. - ArchivedFileInfo *fileInfo = NEW ArchivedFileInfo; - // TheSuperHackers @fix Mauller 23/04/2025 Create new file handle when necessary to prevent memory leak + ArchivedFileInfo fileInfo; ArchiveFile *archiveFile = NEW Win32BIGFile(filename, AsciiString::TheEmptyString); Bool corrupt = FALSE; @@ -153,9 +152,9 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { filesize = betoh(filesize); fileOffset = betoh(fileOffset); - fileInfo->m_archiveFilename = archiveFileName; - fileInfo->m_offset = fileOffset; - fileInfo->m_size = filesize; + fileInfo.m_archiveFilename = archiveFileName; + fileInfo.m_offset = fileOffset; + fileInfo.m_size = filesize; // read in the path name of the file. // the name must terminate within the buffer @@ -179,8 +178,8 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { --filenameIndex; } - fileInfo->m_filename = (char *)(buffer + filenameIndex + 1); - fileInfo->m_filename.toLower(); + fileInfo.m_filename = (char *)(buffer + filenameIndex + 1); + fileInfo.m_filename.toLower(); buffer[filenameIndex + 1] = 0; AsciiString path; @@ -188,16 +187,15 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { AsciiString debugpath; debugpath = path; - debugpath.concat(fileInfo->m_filename); + debugpath.concat(fileInfo.m_filename); // DEBUG_LOG(("Win32BIGFileSystem::openArchiveFile - adding file %s to archive file %s, file number %d", debugpath.str(), fileInfo->m_archiveFilename.str(), i)); - archiveFile->addFile(path, fileInfo); + archiveFile->addFile(path, &fileInfo); } if (corrupt) { DEBUG_CRASH(("Truncated or corrupt directory in BIG file %s", filename)); delete archiveFile; - delete fileInfo; fp->close(); fp = nullptr; return nullptr; @@ -205,9 +203,6 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { archiveFile->attachFile(fp); - delete fileInfo; - fileInfo = nullptr; - // leave fp open as the archive file will be using it. return archiveFile; From ad86b361d5b9086ed3680bcc572e7adfc574eab9 Mon Sep 17 00:00:00 2001 From: xezon <4720891+xezon@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:58:22 +0200 Subject: [PATCH 04/17] refactor(archive): Simplify code in ArchiveFile (#3376) --- Core/GameEngine/Include/Common/ArchiveFile.h | 5 ++- .../Include/Common/ArchiveFileSystem.h | 4 +- .../Source/Common/System/ArchiveFile.cpp | 42 +++++++------------ 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/Core/GameEngine/Include/Common/ArchiveFile.h b/Core/GameEngine/Include/Common/ArchiveFile.h index 63987f9c9cb..4ea8f0c3214 100644 --- a/Core/GameEngine/Include/Common/ArchiveFile.h +++ b/Core/GameEngine/Include/Common/ArchiveFile.h @@ -60,13 +60,16 @@ class ArchiveFile void attachFile(File *file); void getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; - void getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; void addFile(const AsciiString& path, const ArchivedFileInfo *fileInfo); ///< add this file to our directory tree. protected: + void getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; const ArchivedFileInfo * getArchivedFileInfo(const AsciiString& filename) const; ///< return the ArchivedFileInfo from the directory tree. +protected: File *m_file; ///< file pointer to the archive file on disk. Kept open so we don't have to continuously open and close the file all the time. + +private: DetailedArchivedDirectoryInfo m_rootDirectory; }; diff --git a/Core/GameEngine/Include/Common/ArchiveFileSystem.h b/Core/GameEngine/Include/Common/ArchiveFileSystem.h index 48d144484c6..5583290bce0 100644 --- a/Core/GameEngine/Include/Common/ArchiveFileSystem.h +++ b/Core/GameEngine/Include/Common/ArchiveFileSystem.h @@ -143,9 +143,6 @@ class ArchiveFileSystem : public SubsystemInterface virtual Bool loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) = 0; - // Unprotected this for copy-protection routines - ArchiveFile* getArchiveFile(const AsciiString& filename, FileInstance instance = 0) const; - void loadMods(); ArchivedDirectoryInfo* friend_getArchivedDirectoryInfo(const Char* directory); @@ -160,6 +157,7 @@ class ArchiveFileSystem : public SubsystemInterface AsciiString lastToken; ///< Synonymous for file name if the search directory was a file path }; + ArchiveFile* getArchiveFile(const AsciiString& filename, FileInstance instance = 0) const; ArchivedDirectoryInfoResult getArchivedDirectoryInfo(const Char* directory); virtual void loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool overwrite = FALSE, Bool sortedByName = FALSE); ///< load the archive file's header information and apply it to the global archive directory tree. sortedByName inserts by archive filename order (addon number convention), ignoring overwrite. diff --git a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp index 1f982b65a1d..8ca721021d4 100644 --- a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp +++ b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp @@ -106,13 +106,11 @@ void ArchiveFile::addFile(const AsciiString& path, const ArchivedFileInfo *fileI while (!token.isEmpty()) { DetailedArchivedDirectoryInfoMap::iterator tempiter = dirInfo->m_directories.find(token); - if (tempiter == dirInfo->m_directories.end()) - { + if (tempiter == dirInfo->m_directories.end()) { dirInfo = &(dirInfo->m_directories[token]); dirInfo->m_directoryName = token; } - else - { + else { dirInfo = &tempiter->second; } @@ -132,18 +130,12 @@ void ArchiveFile::getFileListInDirectory(const AsciiString& currentDirectory, co tokenizer.nextToken(&token, "\\/"); while (!token.isEmpty()) { - DetailedArchivedDirectoryInfoMap::const_iterator it = dirInfo->m_directories.find(token); - if (it != dirInfo->m_directories.end()) - { - dirInfo = &it->second; - } - else - { - // if the directory doesn't exist, then there aren't any files to be had. + // if the directory doesn't exist, then there aren't any files to be had. + if (it == dirInfo->m_directories.end()) return; - } + dirInfo = &it->second; tokenizer.nextToken(&token, "\\/"); } @@ -153,33 +145,31 @@ void ArchiveFile::getFileListInDirectory(const AsciiString& currentDirectory, co void ArchiveFile::getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const { DetailedArchivedDirectoryInfoMap::const_iterator diriter = dirInfo->m_directories.begin(); - while (diriter != dirInfo->m_directories.end()) { + for (; diriter != dirInfo->m_directories.end(); ++diriter) { const DetailedArchivedDirectoryInfo *tempDirInfo = &(diriter->second); - AsciiString tempdirname; - tempdirname = currentDirectory; - if ((!tempdirname.isEmpty()) && (!tempdirname.endsWith("\\"))) { - tempdirname.concat('\\'); + AsciiString tempDirName = currentDirectory; + if (!tempDirName.isEmpty() && !tempDirName.endsWith("\\")) { + tempDirName.concat('\\'); } - tempdirname.concat(tempDirInfo->m_directoryName); - getFileListInDirectory(tempDirInfo, tempdirname, searchName, filenameList, searchSubdirectories); - diriter++; + tempDirName.concat(tempDirInfo->m_directoryName); + getFileListInDirectory(tempDirInfo, tempDirName, searchName, filenameList, searchSubdirectories); } ArchivedFileInfoMap::const_iterator fileiter = dirInfo->m_files.begin(); - while (fileiter != dirInfo->m_files.end()) { - if (SearchStringMatches(fileiter->second.m_filename, searchName)) { + for (; fileiter != dirInfo->m_files.end(); ++fileiter) { + const ArchivedFileInfo &fileInfo = fileiter->second; + if (SearchStringMatches(fileInfo.m_filename, searchName)) { AsciiString tempfilename; tempfilename = currentDirectory; - if ((!tempfilename.isEmpty()) && (!tempfilename.endsWith("\\"))) { + if (!tempfilename.isEmpty() && !tempfilename.endsWith("\\")) { tempfilename.concat('\\'); } - tempfilename.concat(fileiter->second.m_filename); + tempfilename.concat(fileInfo.m_filename); if (filenameList.find(tempfilename) == filenameList.end()) { // only insert into the list if its not already in there. filenameList.insert(tempfilename); } } - fileiter++; } } From c1a30d51dd216513d172c6846896a304b8914e2d Mon Sep 17 00:00:00 2001 From: xezon <4720891+xezon@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:48:26 +0200 Subject: [PATCH 05/17] fix(filesystem): Fix file path tokenization in ArchiveFile, ArchiveFileSystem, Win32LocalFileSystem (#3374) --- Core/GameEngine/Source/Common/System/ArchiveFile.cpp | 2 +- .../Source/Common/System/ArchiveFileSystem.cpp | 12 ++++++------ .../Win32Device/Common/Win32LocalFileSystem.cpp | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp index 8ca721021d4..798e3ac841e 100644 --- a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp +++ b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp @@ -191,7 +191,7 @@ const ArchivedFileInfo * ArchiveFile::getArchivedFileInfo(const AsciiString& fil tokenizer.toLower(); tokenizer.nextToken(&token, "\\/"); - while (!token.find('.') || tokenizer.find('.')) + while (!tokenizer.isEmpty()) { DetailedArchivedDirectoryInfoMap::const_iterator it = dirInfo->m_directories.find(token); if (it != dirInfo->m_directories.end()) diff --git a/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp b/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp index 5720c5b9417..17d49e8b372 100644 --- a/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp +++ b/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp @@ -144,9 +144,9 @@ void ArchiveFileSystem::loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool ove AsciiString token; AsciiString tokenizer = *it; tokenizer.toLower(); - Bool infoInPath = tokenizer.nextToken(&token, "\\/"); + tokenizer.nextToken(&token, "\\/"); - while (infoInPath && (!token.find('.') || tokenizer.find('.'))) + while (!tokenizer.isEmpty()) { path.concat(token); path.concat('\\'); @@ -163,7 +163,7 @@ void ArchiveFileSystem::loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool ove dirInfo = &tempiter->second; } - infoInPath = tokenizer.nextToken(&token, "\\/"); + tokenizer.nextToken(&token, "\\/"); } ArchivedFileLocationMap::iterator fileIt; @@ -306,15 +306,15 @@ ArchiveFileSystem::ArchivedDirectoryInfoResult ArchiveFileSystem::getArchivedDir AsciiString token; AsciiString tokenizer = directory; tokenizer.toLower(); - Bool infoInPath = tokenizer.nextToken(&token, "\\/"); + tokenizer.nextToken(&token, "\\/"); - while (infoInPath && (!token.find('.') || tokenizer.find('.'))) + while (!tokenizer.isEmpty()) { ArchivedDirectoryInfoMap::iterator tempiter = dirInfo->m_directories.find(token); if (tempiter != dirInfo->m_directories.end()) { dirInfo = &tempiter->second; - infoInPath = tokenizer.nextToken(&token, "\\/"); + tokenizer.nextToken(&token, "\\/"); } else { diff --git a/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp b/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp index 43a6ca5cbdd..a4ddcb6b59f 100644 --- a/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp +++ b/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp @@ -60,7 +60,7 @@ File * Win32LocalFileSystem::openFile(const Char *filename, Int access, size_t b AsciiString dirName; string.nextToken(&token, "\\/"); dirName = token; - while ((token.find('.') == nullptr) || (string.find('.') != nullptr)) { + while (!string.isEmpty()) { createDirectory(dirName); string.nextToken(&token, "\\/"); dirName.concat('\\'); From 877c621857262dd552a262df317b350f78035dc5 Mon Sep 17 00:00:00 2001 From: xezon <4720891+xezon@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:32:53 +0200 Subject: [PATCH 06/17] perf(archivefile): Take timestamps from cached file info in Win32BIGFile::getFileInfo(), StdBIGFile::getFileInfo() (#3415) --- Core/GameEngine/Include/Common/ArchiveFile.h | 1 + Core/GameEngine/Include/Common/FileSystem.h | 8 ++++++++ .../GameEngine/Source/Common/System/ArchiveFile.cpp | 12 ++++++++++++ .../Source/StdDevice/Common/StdBIGFile.cpp | 13 +++++++------ .../Source/StdDevice/Common/StdLocalFileSystem.cpp | 4 ++-- .../Source/Win32Device/Common/Win32BIGFile.cpp | 13 +++++++------ .../Win32Device/Common/Win32LocalFileSystem.cpp | 4 ++-- 7 files changed, 39 insertions(+), 16 deletions(-) diff --git a/Core/GameEngine/Include/Common/ArchiveFile.h b/Core/GameEngine/Include/Common/ArchiveFile.h index 4ea8f0c3214..b9d4e085567 100644 --- a/Core/GameEngine/Include/Common/ArchiveFile.h +++ b/Core/GameEngine/Include/Common/ArchiveFile.h @@ -69,6 +69,7 @@ class ArchiveFile protected: File *m_file; ///< file pointer to the archive file on disk. Kept open so we don't have to continuously open and close the file all the time. + FileInfo m_fileInfo; ///< file info of the archive file on disk. private: DetailedArchivedDirectoryInfo m_rootDirectory; diff --git a/Core/GameEngine/Include/Common/FileSystem.h b/Core/GameEngine/Include/Common/FileSystem.h index 8a8b99a9f8c..75f93caabb5 100644 --- a/Core/GameEngine/Include/Common/FileSystem.h +++ b/Core/GameEngine/Include/Common/FileSystem.h @@ -109,6 +109,14 @@ typedef UnsignedByte FileInstance; struct FileInfo { + FileInfo() + : sizeHigh(0) + , sizeLow(0) + , timestampHigh(0) + , timestampLow(0) + { + } + Int64 size() const { return (Int64)sizeHigh << 32 | sizeLow; } Int64 timestamp() const { return (Int64)timestampHigh << 32 | timestampLow; } diff --git a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp index 798e3ac841e..e5a6e81d329 100644 --- a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp +++ b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp @@ -30,6 +30,7 @@ #include "Common/ArchiveFile.h" #include "Common/ArchiveFileSystem.h" +#include "Common/LocalFileSystem.h" #include "Common/file.h" #include "Common/PerfTimer.h" @@ -179,7 +180,18 @@ void ArchiveFile::attachFile(File *file) m_file->close(); m_file = nullptr; } + m_file = file; + + if (m_file != nullptr) { + if (!TheLocalFileSystem->getFileInfo(m_file->getName(), &m_fileInfo)) { + DEBUG_CRASH(("ArchiveFile::attachFile - Cannot query file info for '%s' unexpectedly", m_file->getName())); + m_fileInfo = FileInfo(); + } + } + else { + m_fileInfo = FileInfo(); + } } const ArchivedFileInfo * ArchiveFile::getArchivedFileInfo(const AsciiString& filename) const diff --git a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFile.cpp b/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFile.cpp index c873b733524..5f063c2869c 100644 --- a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFile.cpp +++ b/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFile.cpp @@ -151,17 +151,18 @@ void StdBIGFile::close() Bool StdBIGFile::getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const { - const ArchivedFileInfo *tempFileInfo = getArchivedFileInfo(filename); + const ArchivedFileInfo *archivedFileInfo = getArchivedFileInfo(filename); - if (tempFileInfo == nullptr) { + if (archivedFileInfo == nullptr) { return FALSE; } - TheLocalFileSystem->getFileInfo(AsciiString(m_file->getName()), fileInfo); - - // fill in the size info. Since the size can't be bigger than a JUNK file, the high Int will always be 0. + // fill in the size info. Since the size can't be bigger than a JUNK file, the high int will always be 0. fileInfo->sizeHigh = 0; - fileInfo->sizeLow = tempFileInfo->m_size; + fileInfo->sizeLow = archivedFileInfo->m_size; + // the timestamp is taken from the archive file. + fileInfo->timestampHigh = m_fileInfo.timestampHigh; + fileInfo->timestampLow = m_fileInfo.timestampLow; return TRUE; } diff --git a/Core/GameEngineDevice/Source/StdDevice/Common/StdLocalFileSystem.cpp b/Core/GameEngineDevice/Source/StdDevice/Common/StdLocalFileSystem.cpp index d47e473f4d2..ee53db20549 100644 --- a/Core/GameEngineDevice/Source/StdDevice/Common/StdLocalFileSystem.cpp +++ b/Core/GameEngineDevice/Source/StdDevice/Common/StdLocalFileSystem.cpp @@ -304,10 +304,10 @@ Bool StdLocalFileSystem::getFileInfo(const AsciiString& filename, FileInfo *file // TODO: fix this to be win compatible (time since 1601) auto time = write_time.time_since_epoch().count(); + fileInfo->sizeHigh = file_size >> 32; + fileInfo->sizeLow = file_size & UINT32_MAX; fileInfo->timestampHigh = time >> 32; fileInfo->timestampLow = time & UINT32_MAX; - fileInfo->sizeHigh = file_size >> 32; - fileInfo->sizeLow = file_size & UINT32_MAX; return TRUE; } diff --git a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFile.cpp b/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFile.cpp index 3deee01fd82..fa21e09a278 100644 --- a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFile.cpp +++ b/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFile.cpp @@ -151,17 +151,18 @@ void Win32BIGFile::close() Bool Win32BIGFile::getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const { - const ArchivedFileInfo *tempFileInfo = getArchivedFileInfo(filename); + const ArchivedFileInfo *archivedFileInfo = getArchivedFileInfo(filename); - if (tempFileInfo == nullptr) { + if (archivedFileInfo == nullptr) { return FALSE; } - TheLocalFileSystem->getFileInfo(AsciiString(m_file->getName()), fileInfo); - - // fill in the size info. Since the size can't be bigger than a JUNK file, the high Int will always be 0. + // fill in the size info. Since the size can't be bigger than a JUNK file, the high int will always be 0. fileInfo->sizeHigh = 0; - fileInfo->sizeLow = tempFileInfo->m_size; + fileInfo->sizeLow = archivedFileInfo->m_size; + // the timestamp is taken from the archive file. + fileInfo->timestampHigh = m_fileInfo.timestampHigh; + fileInfo->timestampLow = m_fileInfo.timestampLow; return TRUE; } diff --git a/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp b/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp index a4ddcb6b59f..5ec6fd35680 100644 --- a/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp +++ b/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp @@ -194,10 +194,10 @@ Bool Win32LocalFileSystem::getFileInfo(const AsciiString& filename, FileInfo *fi return FALSE; } - fileInfo->timestampHigh = findData.ftLastWriteTime.dwHighDateTime; - fileInfo->timestampLow = findData.ftLastWriteTime.dwLowDateTime; fileInfo->sizeHigh = findData.nFileSizeHigh; fileInfo->sizeLow = findData.nFileSizeLow; + fileInfo->timestampHigh = findData.ftLastWriteTime.dwHighDateTime; + fileInfo->timestampLow = findData.ftLastWriteTime.dwLowDateTime; FindClose(findHandle); From 7c9b184c9b0da385beefe34b8b0ab3908913087f Mon Sep 17 00:00:00 2001 From: xezon <4720891+xezon@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:08:58 +0200 Subject: [PATCH 07/17] refactor(archivefile): Merge Win32BIGFile and StdBIGFile into ArchiveFile (#3505) --- Core/GameEngine/Include/Common/ArchiveFile.h | 20 ++- .../Source/Common/System/ArchiveFile.cpp | 120 ++++++++++++- Core/GameEngineDevice/CMakeLists.txt | 4 - .../Include/StdDevice/Common/StdBIGFile.h | 53 ------ .../Include/Win32Device/Common/Win32BIGFile.h | 53 ------ .../Source/StdDevice/Common/StdBIGFile.cpp | 169 ------------------ .../StdDevice/Common/StdBIGFileSystem.cpp | 3 +- .../Win32Device/Common/Win32BIGFile.cpp | 169 ------------------ .../Win32Device/Common/Win32BIGFileSystem.cpp | 3 +- Generals/Code/GameEngineDevice/CMakeLists.txt | 2 - .../Code/GameEngineDevice/CMakeLists.txt | 2 - 11 files changed, 132 insertions(+), 466 deletions(-) delete mode 100644 Core/GameEngineDevice/Include/StdDevice/Common/StdBIGFile.h delete mode 100644 Core/GameEngineDevice/Include/Win32Device/Common/Win32BIGFile.h delete mode 100644 Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFile.cpp delete mode 100644 Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFile.cpp diff --git a/Core/GameEngine/Include/Common/ArchiveFile.h b/Core/GameEngine/Include/Common/ArchiveFile.h index b9d4e085567..a7e0ed68bfb 100644 --- a/Core/GameEngine/Include/Common/ArchiveFile.h +++ b/Core/GameEngine/Include/Common/ArchiveFile.h @@ -47,16 +47,16 @@ class File; class ArchiveFile { public: - ArchiveFile(); - virtual ~ArchiveFile(); + ArchiveFile(AsciiString name, AsciiString path); + ~ArchiveFile(); - virtual Bool getFileInfo( const AsciiString& filename, FileInfo *fileInfo) const = 0; ///< fill in the fileInfo struct with info about the file requested. - virtual File* openFile( const Char *filename, Int access = 0) = 0; ///< Open the specified file within the archive file - virtual void closeAllFiles() = 0; ///< Close all file opened in this archive file - virtual AsciiString getName() = 0; ///< Returns the name of the archive file - virtual AsciiString getPath() = 0; ///< Returns full path and name of archive file - virtual void setSearchPriority( Int new_priority ) = 0; ///< Set this archive file's search priority - virtual void close() = 0; ///< Close this archive file + Bool getFileInfo( const AsciiString& filename, FileInfo *fileInfo) const; ///< fill in the fileInfo struct with info about the file requested. + File* openFile( const Char *filename, Int access = 0); ///< Open the specified file within the archive file + void closeAllFiles(); ///< Close all file opened in this archive file + AsciiString getName(); ///< Returns the name of the archive file + AsciiString getPath(); ///< Returns full path and name of archive file + void setSearchPriority( Int new_priority ); ///< Set this archive file's search priority + void close(); ///< Close this archive file void attachFile(File *file); void getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; @@ -70,6 +70,8 @@ class ArchiveFile protected: File *m_file; ///< file pointer to the archive file on disk. Kept open so we don't have to continuously open and close the file all the time. FileInfo m_fileInfo; ///< file info of the archive file on disk. + AsciiString m_name; ///< archive file name + AsciiString m_path; ///< archive file path private: DetailedArchivedDirectoryInfo m_rootDirectory; diff --git a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp index e5a6e81d329..9e0db3f0ee2 100644 --- a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp +++ b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp @@ -32,7 +32,10 @@ #include "Common/ArchiveFileSystem.h" #include "Common/LocalFileSystem.h" #include "Common/file.h" +#include "Common/LocalFile.h" #include "Common/PerfTimer.h" +#include "Common/RAMFile.h" +#include "Common/StreamingArchiveFile.h" // checks to see if str matches searchString. Search string is done in the @@ -90,8 +93,10 @@ ArchiveFile::~ArchiveFile() } } -ArchiveFile::ArchiveFile() +ArchiveFile::ArchiveFile(AsciiString name, AsciiString path) : m_file(nullptr) + , m_name(name) + , m_path(path) { } @@ -229,3 +234,116 @@ const ArchivedFileInfo * ArchiveFile::getArchivedFileInfo(const AsciiString& fil } } + +//============================================================================ +// ArchiveFile::openFile +//============================================================================ + +File* ArchiveFile::openFile( const Char *filename, Int access ) +{ + const ArchivedFileInfo *fileInfo = getArchivedFileInfo(AsciiString(filename)); + + if (fileInfo == nullptr) { + return nullptr; + } + + RAMFile *ramFile = nullptr; + + if (BitIsSet(access, File::STREAMING)) + ramFile = newInstance( StreamingArchiveFile ); + else + ramFile = newInstance( RAMFile ); + + ramFile->deleteOnClose(); + if (ramFile->openFromArchive(m_file, fileInfo->m_filename, fileInfo->m_offset, fileInfo->m_size) == FALSE) { + ramFile->close(); + ramFile = nullptr; + return nullptr; + } + + if ((access & File::WRITE) == 0) { + // requesting read only access. Just return the RAM file. + return ramFile; + } + + // whoever is opening this file wants write access, so copy the file to the local disk + // and return that file pointer. + + constexpr size_t bufferSize = 0; + File *localFile = TheLocalFileSystem->openFile(filename, access, bufferSize); + if (localFile != nullptr) { + ramFile->copyDataToFile(localFile); + } + + ramFile->close(); + ramFile = nullptr; + + return localFile; +} + +//============================================================================ +// ArchiveFile::closeAllFiles +//============================================================================ + +void ArchiveFile::closeAllFiles() +{ + +} + +//============================================================================ +// ArchiveFile::getName +//============================================================================ + +AsciiString ArchiveFile::getName() +{ + return m_name; +} + +//============================================================================ +// ArchiveFile::getPath +//============================================================================ + +AsciiString ArchiveFile::getPath() +{ + return m_path; +} + +//============================================================================ +// ArchiveFile::setSearchPriority +//============================================================================ + +void ArchiveFile::setSearchPriority( Int new_priority ) +{ + +} + +//============================================================================ +// ArchiveFile::close +//============================================================================ + +void ArchiveFile::close() +{ + +} + +//============================================================================ +// ArchiveFile::getFileInfo +//============================================================================ + +Bool ArchiveFile::getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const +{ + const ArchivedFileInfo *archivedFileInfo = getArchivedFileInfo(filename); + + if (archivedFileInfo == nullptr) { + return FALSE; + } + + // fill in the size info. Since the size can't be bigger than a JUNK file, the high int will always be 0. + fileInfo->sizeHigh = 0; + fileInfo->sizeLow = archivedFileInfo->m_size; + // the timestamp is taken from the archive file. + fileInfo->timestampHigh = m_fileInfo.timestampHigh; + fileInfo->timestampLow = m_fileInfo.timestampLow; + + return TRUE; +} diff --git a/Core/GameEngineDevice/CMakeLists.txt b/Core/GameEngineDevice/CMakeLists.txt index 6added5403b..3315487c387 100644 --- a/Core/GameEngineDevice/CMakeLists.txt +++ b/Core/GameEngineDevice/CMakeLists.txt @@ -83,7 +83,6 @@ set(GAMEENGINEDEVICE_SRC # Include/W3DDevice/GameLogic/W3DGameLogic.h # Include/W3DDevice/GameLogic/W3DGhostObject.h # Include/W3DDevice/GameLogic/W3DTerrainLogic.h - Include/Win32Device/Common/Win32BIGFile.h Include/Win32Device/Common/Win32BIGFileSystem.h # Include/Win32Device/Common/Win32GameEngine.h Include/Win32Device/Common/Win32LocalFile.h @@ -186,7 +185,6 @@ set(GAMEENGINEDEVICE_SRC # Source/W3DDevice/GameLogic/W3DGameLogic.cpp # Source/W3DDevice/GameLogic/W3DGhostObject.cpp # Source/W3DDevice/GameLogic/W3DTerrainLogic.cpp - Source/Win32Device/Common/Win32BIGFile.cpp Source/Win32Device/Common/Win32BIGFileSystem.cpp # Source/Win32Device/Common/Win32GameEngine.cpp Source/Win32Device/Common/Win32LocalFile.cpp @@ -200,11 +198,9 @@ set(GAMEENGINEDEVICE_SRC # Add C++ 17 FileSystem implementation for non-VS6 builds if(NOT IS_VS6_BUILD) list(APPEND GAMEENGINEDEVICE_SRC - Include/StdDevice/Common/StdBIGFile.h Include/StdDevice/Common/StdBIGFileSystem.h Include/StdDevice/Common/StdLocalFile.h Include/StdDevice/Common/StdLocalFileSystem.h - Source/StdDevice/Common/StdBIGFile.cpp Source/StdDevice/Common/StdBIGFileSystem.cpp Source/StdDevice/Common/StdLocalFile.cpp Source/StdDevice/Common/StdLocalFileSystem.cpp diff --git a/Core/GameEngineDevice/Include/StdDevice/Common/StdBIGFile.h b/Core/GameEngineDevice/Include/StdDevice/Common/StdBIGFile.h deleted file mode 100644 index 36538853352..00000000000 --- a/Core/GameEngineDevice/Include/StdDevice/Common/StdBIGFile.h +++ /dev/null @@ -1,53 +0,0 @@ -/* -** Command & Conquer Generals Zero Hour(tm) -** Copyright 2025 Electronic Arts Inc. -** -** This program is free software: you can redistribute it and/or modify -** it under the terms of the GNU General Public License as published by -** the Free Software Foundation, either version 3 of the License, or -** (at your option) any later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program. If not, see . -*/ - -//////////////////////////////////////////////////////////////////////////////// -// // -// (c) 2001-2003 Electronic Arts Inc. // -// // -//////////////////////////////////////////////////////////////////////////////// - -/////// StdBIGFile.h //////////////////////////////////// -// Stephan Vedder, April 2025 -/////////////////////////////////////////////////////////// - -#pragma once - -#include "Common/ArchiveFile.h" -#include "Common/AsciiString.h" -#include "Common/List.h" - -class StdBIGFile : public ArchiveFile -{ - public: - StdBIGFile(AsciiString name, AsciiString path); - virtual ~StdBIGFile() override; - - virtual Bool getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const override; ///< fill in the fileInfo struct with info about the requested file. - virtual File* openFile( const Char *filename, Int access = 0 ) override;///< Open the specified file within the BIG file - virtual void closeAllFiles() override; ///< Close all file opened in this BIG file - virtual AsciiString getName() override; ///< Returns the name of the BIG file - virtual AsciiString getPath() override; ///< Returns full path and name of BIG file - virtual void setSearchPriority( Int new_priority ) override; ///< Set this BIG file's search priority - virtual void close() override; ///< Close this BIG file - - protected: - - AsciiString m_name; ///< BIG file name - AsciiString m_path; ///< BIG file path -}; diff --git a/Core/GameEngineDevice/Include/Win32Device/Common/Win32BIGFile.h b/Core/GameEngineDevice/Include/Win32Device/Common/Win32BIGFile.h deleted file mode 100644 index 093aadbcf4c..00000000000 --- a/Core/GameEngineDevice/Include/Win32Device/Common/Win32BIGFile.h +++ /dev/null @@ -1,53 +0,0 @@ -/* -** Command & Conquer Generals Zero Hour(tm) -** Copyright 2025 Electronic Arts Inc. -** -** This program is free software: you can redistribute it and/or modify -** it under the terms of the GNU General Public License as published by -** the Free Software Foundation, either version 3 of the License, or -** (at your option) any later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program. If not, see . -*/ - -//////////////////////////////////////////////////////////////////////////////// -// // -// (c) 2001-2003 Electronic Arts Inc. // -// // -//////////////////////////////////////////////////////////////////////////////// - -/////// Win32BIGFile.h //////////////////////////////////// -// Bryan Cleveland, August 2002 -/////////////////////////////////////////////////////////// - -#pragma once - -#include "Common/ArchiveFile.h" -#include "Common/AsciiString.h" -#include "Common/List.h" - -class Win32BIGFile : public ArchiveFile -{ - public: - Win32BIGFile(AsciiString name, AsciiString path); - virtual ~Win32BIGFile() override; - - virtual Bool getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const override; ///< fill in the fileInfo struct with info about the requested file. - virtual File* openFile( const Char *filename, Int access = 0 ) override;///< Open the specified file within the BIG file - virtual void closeAllFiles() override; ///< Close all file opened in this BIG file - virtual AsciiString getName() override; ///< Returns the name of the BIG file - virtual AsciiString getPath() override; ///< Returns full path and name of BIG file - virtual void setSearchPriority( Int new_priority ) override; ///< Set this BIG file's search priority - virtual void close() override; ///< Close this BIG file - - protected: - - AsciiString m_name; ///< BIG file name - AsciiString m_path; ///< BIG file path -}; diff --git a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFile.cpp b/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFile.cpp deleted file mode 100644 index 5f063c2869c..00000000000 --- a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFile.cpp +++ /dev/null @@ -1,169 +0,0 @@ -/* -** Command & Conquer Generals Zero Hour(tm) -** Copyright 2025 Electronic Arts Inc. -** -** This program is free software: you can redistribute it and/or modify -** it under the terms of the GNU General Public License as published by -** the Free Software Foundation, either version 3 of the License, or -** (at your option) any later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program. If not, see . -*/ - -//////////////////////////////////////////////////////////////////////////////// -// // -// (c) 2001-2003 Electronic Arts Inc. // -// // -//////////////////////////////////////////////////////////////////////////////// - -////// StdBIGFile.cpp ///////////////////////// -// Stephan Vedder, April 2025 -///////////////////////////////////////////////////// - -#include "Common/LocalFile.h" -#include "Common/LocalFileSystem.h" -#include "Common/RAMFile.h" -#include "Common/StreamingArchiveFile.h" -#include "Common/GameMemory.h" -#include "Common/PerfTimer.h" -#include "StdDevice/Common/StdBIGFile.h" - -//============================================================================ -// StdBIGFile::StdBIGFile -//============================================================================ - -StdBIGFile::StdBIGFile(AsciiString name, AsciiString path) - : m_name(name) - , m_path(path) -{ - -} - -//============================================================================ -// StdBIGFile::~StdBIGFile -//============================================================================ - -StdBIGFile::~StdBIGFile() -{ - -} - -//============================================================================ -// StdBIGFile::openFile -//============================================================================ - -File* StdBIGFile::openFile( const Char *filename, Int access ) -{ - const ArchivedFileInfo *fileInfo = getArchivedFileInfo(AsciiString(filename)); - - if (fileInfo == nullptr) { - return nullptr; - } - - RAMFile *ramFile = nullptr; - - if (BitIsSet(access, File::STREAMING)) - ramFile = newInstance( StreamingArchiveFile ); - else - ramFile = newInstance( RAMFile ); - - ramFile->deleteOnClose(); - if (ramFile->openFromArchive(m_file, fileInfo->m_filename, fileInfo->m_offset, fileInfo->m_size) == FALSE) { - ramFile->close(); - ramFile = nullptr; - return nullptr; - } - - if ((access & File::WRITE) == 0) { - // requesting read only access. Just return the RAM file. - return ramFile; - } - - // whoever is opening this file wants write access, so copy the file to the local disk - // and return that file pointer. - - constexpr size_t bufferSize = 0; - File *localFile = TheLocalFileSystem->openFile(filename, access, bufferSize); - if (localFile != nullptr) { - ramFile->copyDataToFile(localFile); - } - - ramFile->close(); - ramFile = nullptr; - - return localFile; -} - -//============================================================================ -// StdBIGFile::closeAllFiles -//============================================================================ - -void StdBIGFile::closeAllFiles() -{ - -} - -//============================================================================ -// StdBIGFile::getName -//============================================================================ - -AsciiString StdBIGFile::getName() -{ - return m_name; -} - -//============================================================================ -// StdBIGFile::getPath -//============================================================================ - -AsciiString StdBIGFile::getPath() -{ - return m_path; -} - -//============================================================================ -// StdBIGFile::setSearchPriority -//============================================================================ - -void StdBIGFile::setSearchPriority( Int new_priority ) -{ - -} - -//============================================================================ -// StdBIGFile::close -//============================================================================ - -void StdBIGFile::close() -{ - -} - -//============================================================================ -// StdBIGFile::getFileInfo -//============================================================================ - -Bool StdBIGFile::getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const -{ - const ArchivedFileInfo *archivedFileInfo = getArchivedFileInfo(filename); - - if (archivedFileInfo == nullptr) { - return FALSE; - } - - // fill in the size info. Since the size can't be bigger than a JUNK file, the high int will always be 0. - fileInfo->sizeHigh = 0; - fileInfo->sizeLow = archivedFileInfo->m_size; - // the timestamp is taken from the archive file. - fileInfo->timestampHigh = m_fileInfo.timestampHigh; - fileInfo->timestampLow = m_fileInfo.timestampLow; - - return TRUE; -} - diff --git a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp b/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp index 0fb035e0c3e..529f5b03648 100644 --- a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp +++ b/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp @@ -38,7 +38,6 @@ #include "Common/Registry.h" #endif -#include "StdDevice/Common/StdBIGFile.h" #include "StdDevice/Common/StdBIGFileSystem.h" #include "Utility/endian_compat.h" @@ -129,7 +128,7 @@ ArchiveFile * StdBIGFileSystem::openArchiveFile(const Char *filename) { fp->seek(0x10, File::START); // read in each directory listing. ArchivedFileInfo fileInfo; - ArchiveFile *archiveFile = NEW StdBIGFile(filename, AsciiString::TheEmptyString); + ArchiveFile *archiveFile = NEW ArchiveFile(filename, AsciiString::TheEmptyString); for (Int i = 0; i < numLittleFiles; ++i) { Int filesize = 0; diff --git a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFile.cpp b/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFile.cpp deleted file mode 100644 index fa21e09a278..00000000000 --- a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFile.cpp +++ /dev/null @@ -1,169 +0,0 @@ -/* -** Command & Conquer Generals Zero Hour(tm) -** Copyright 2025 Electronic Arts Inc. -** -** This program is free software: you can redistribute it and/or modify -** it under the terms of the GNU General Public License as published by -** the Free Software Foundation, either version 3 of the License, or -** (at your option) any later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program. If not, see . -*/ - -//////////////////////////////////////////////////////////////////////////////// -// // -// (c) 2001-2003 Electronic Arts Inc. // -// // -//////////////////////////////////////////////////////////////////////////////// - -////// Win32BIGFile.cpp ///////////////////////// -// Bryan Cleveland, August 2002 -///////////////////////////////////////////////////// - -#include "Common/LocalFile.h" -#include "Common/LocalFileSystem.h" -#include "Common/RAMFile.h" -#include "Common/StreamingArchiveFile.h" -#include "Common/GameMemory.h" -#include "Common/PerfTimer.h" -#include "Win32Device/Common/Win32BIGFile.h" - -//============================================================================ -// Win32BIGFile::Win32BIGFile -//============================================================================ - -Win32BIGFile::Win32BIGFile(AsciiString name, AsciiString path) - : m_name(name) - , m_path(path) -{ - -} - -//============================================================================ -// Win32BIGFile::~Win32BIGFile -//============================================================================ - -Win32BIGFile::~Win32BIGFile() -{ - -} - -//============================================================================ -// Win32BIGFile::openFile -//============================================================================ - -File* Win32BIGFile::openFile( const Char *filename, Int access ) -{ - const ArchivedFileInfo *fileInfo = getArchivedFileInfo(AsciiString(filename)); - - if (fileInfo == nullptr) { - return nullptr; - } - - RAMFile *ramFile = nullptr; - - if (BitIsSet(access, File::STREAMING)) - ramFile = newInstance( StreamingArchiveFile ); - else - ramFile = newInstance( RAMFile ); - - ramFile->deleteOnClose(); - if (ramFile->openFromArchive(m_file, fileInfo->m_filename, fileInfo->m_offset, fileInfo->m_size) == FALSE) { - ramFile->close(); - ramFile = nullptr; - return nullptr; - } - - if ((access & File::WRITE) == 0) { - // requesting read only access. Just return the RAM file. - return ramFile; - } - - // whoever is opening this file wants write access, so copy the file to the local disk - // and return that file pointer. - - constexpr size_t bufferSize = 0; - File *localFile = TheLocalFileSystem->openFile(filename, access, bufferSize); - if (localFile != nullptr) { - ramFile->copyDataToFile(localFile); - } - - ramFile->close(); - ramFile = nullptr; - - return localFile; -} - -//============================================================================ -// Win32BIGFile::closeAllFiles -//============================================================================ - -void Win32BIGFile::closeAllFiles() -{ - -} - -//============================================================================ -// Win32BIGFile::getName -//============================================================================ - -AsciiString Win32BIGFile::getName() -{ - return m_name; -} - -//============================================================================ -// Win32BIGFile::getPath -//============================================================================ - -AsciiString Win32BIGFile::getPath() -{ - return m_path; -} - -//============================================================================ -// Win32BIGFile::setSearchPriority -//============================================================================ - -void Win32BIGFile::setSearchPriority( Int new_priority ) -{ - -} - -//============================================================================ -// Win32BIGFile::close -//============================================================================ - -void Win32BIGFile::close() -{ - -} - -//============================================================================ -// Win32BIGFile::getFileInfo -//============================================================================ - -Bool Win32BIGFile::getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const -{ - const ArchivedFileInfo *archivedFileInfo = getArchivedFileInfo(filename); - - if (archivedFileInfo == nullptr) { - return FALSE; - } - - // fill in the size info. Since the size can't be bigger than a JUNK file, the high int will always be 0. - fileInfo->sizeHigh = 0; - fileInfo->sizeLow = archivedFileInfo->m_size; - // the timestamp is taken from the archive file. - fileInfo->timestampHigh = m_fileInfo.timestampHigh; - fileInfo->timestampLow = m_fileInfo.timestampLow; - - return TRUE; -} - diff --git a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp b/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp index 004e393ec02..2159c367728 100644 --- a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp +++ b/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp @@ -38,7 +38,6 @@ #include "Common/Registry.h" #endif -#include "Win32Device/Common/Win32BIGFile.h" #include "Win32Device/Common/Win32BIGFileSystem.h" #include "Utility/endian_compat.h" @@ -138,7 +137,7 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { fp->seek(0x10, File::START); // read in each directory listing. ArchivedFileInfo fileInfo; - ArchiveFile *archiveFile = NEW Win32BIGFile(filename, AsciiString::TheEmptyString); + ArchiveFile *archiveFile = NEW ArchiveFile(filename, AsciiString::TheEmptyString); Bool corrupt = FALSE; for (Int i = 0; i < numLittleFiles; ++i) { diff --git a/Generals/Code/GameEngineDevice/CMakeLists.txt b/Generals/Code/GameEngineDevice/CMakeLists.txt index dea3ff6c9ae..bedd7c48520 100644 --- a/Generals/Code/GameEngineDevice/CMakeLists.txt +++ b/Generals/Code/GameEngineDevice/CMakeLists.txt @@ -70,7 +70,6 @@ set(GAMEENGINEDEVICE_SRC Include/W3DDevice/GameLogic/W3DGameLogic.h Include/W3DDevice/GameLogic/W3DGhostObject.h Include/W3DDevice/GameLogic/W3DTerrainLogic.h -# Include/Win32Device/Common/Win32BIGFile.h # Include/Win32Device/Common/Win32BIGFileSystem.h Include/Win32Device/Common/Win32GameEngine.h # Include/Win32Device/Common/Win32LocalFile.h @@ -168,7 +167,6 @@ set(GAMEENGINEDEVICE_SRC Source/W3DDevice/GameLogic/W3DGameLogic.cpp Source/W3DDevice/GameLogic/W3DGhostObject.cpp Source/W3DDevice/GameLogic/W3DTerrainLogic.cpp -# Source/Win32Device/Common/Win32BIGFile.cpp # Source/Win32Device/Common/Win32BIGFileSystem.cpp Source/Win32Device/Common/Win32GameEngine.cpp # Source/Win32Device/Common/Win32LocalFile.cpp diff --git a/GeneralsMD/Code/GameEngineDevice/CMakeLists.txt b/GeneralsMD/Code/GameEngineDevice/CMakeLists.txt index 56408e821bd..e8567307646 100644 --- a/GeneralsMD/Code/GameEngineDevice/CMakeLists.txt +++ b/GeneralsMD/Code/GameEngineDevice/CMakeLists.txt @@ -81,7 +81,6 @@ set(GAMEENGINEDEVICE_SRC Include/W3DDevice/GameLogic/W3DGameLogic.h Include/W3DDevice/GameLogic/W3DGhostObject.h Include/W3DDevice/GameLogic/W3DTerrainLogic.h -# Include/Win32Device/Common/Win32BIGFile.h # Include/Win32Device/Common/Win32BIGFileSystem.h Include/Win32Device/Common/Win32GameEngine.h # Include/Win32Device/Common/Win32LocalFile.h @@ -181,7 +180,6 @@ set(GAMEENGINEDEVICE_SRC Source/W3DDevice/GameLogic/W3DGameLogic.cpp Source/W3DDevice/GameLogic/W3DGhostObject.cpp Source/W3DDevice/GameLogic/W3DTerrainLogic.cpp -# Source/Win32Device/Common/Win32BIGFile.cpp # Source/Win32Device/Common/Win32BIGFileSystem.cpp Source/Win32Device/Common/Win32GameEngine.cpp # Source/Win32Device/Common/Win32LocalFile.cpp From 13ec7acdcf3a04f100ec3b0f294ef8da05e2d7b2 Mon Sep 17 00:00:00 2001 From: xezon <4720891+xezon@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:10:57 +0200 Subject: [PATCH 08/17] refactor(archivefilesystem): Merge Win32BIGFileSystem and StdBIGFileSystem into BIGFileSystem (#3505) --- Core/GameEngine/CMakeLists.txt | 2 + .../Include/Common/BIGFileSystem.h} | 8 +- .../Source/Common/System/BIGFileSystem.cpp} | 42 +-- Core/GameEngineDevice/CMakeLists.txt | 4 - .../StdDevice/Common/StdBIGFileSystem.h | 55 ---- .../StdDevice/Common/StdBIGFileSystem.cpp | 239 ------------------ Core/Tools/Autorun/autorun.cpp | 4 +- Core/Tools/MapCacheBuilder/Source/WinMain.cpp | 5 +- Generals/Code/GameEngineDevice/CMakeLists.txt | 2 - .../Win32Device/Common/Win32GameEngine.h | 4 +- .../Code/Tools/GUIEdit/Source/GUIEdit.cpp | 4 +- .../Tools/WorldBuilder/src/WorldBuilder.cpp | 4 +- .../Code/GameEngineDevice/CMakeLists.txt | 2 - .../Win32Device/Common/Win32GameEngine.h | 4 +- .../Code/Tools/GUIEdit/Source/GUIEdit.cpp | 4 +- .../Tools/WorldBuilder/src/WorldBuilder.cpp | 4 +- 16 files changed, 44 insertions(+), 343 deletions(-) rename Core/{GameEngineDevice/Include/Win32Device/Common/Win32BIGFileSystem.h => GameEngine/Include/Common/BIGFileSystem.h} (91%) rename Core/{GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp => GameEngine/Source/Common/System/BIGFileSystem.cpp} (82%) delete mode 100644 Core/GameEngineDevice/Include/StdDevice/Common/StdBIGFileSystem.h delete mode 100644 Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp diff --git a/Core/GameEngine/CMakeLists.txt b/Core/GameEngine/CMakeLists.txt index e6aee6da85f..8f5892206e0 100644 --- a/Core/GameEngine/CMakeLists.txt +++ b/Core/GameEngine/CMakeLists.txt @@ -15,6 +15,7 @@ set(GAMEENGINE_SRC # Include/Common/BattleHonors.h # Include/Common/BezFwdIterator.h # Include/Common/BezierSegment.h + Include/Common/BIGFileSystem.h # Include/Common/BitFlags.h # Include/Common/BitFlagsIO.h # Include/Common/BorderColors.h @@ -644,6 +645,7 @@ set(GAMEENGINE_SRC Source/Common/System/ArchiveFile.cpp Source/Common/System/ArchiveFileSystem.cpp Source/Common/System/AsciiString.cpp + Source/Common/System/BIGFileSystem.cpp # Source/Common/System/BuildAssistant.cpp # Source/Common/System/CriticalSection.cpp # Source/Common/System/DataChunk.cpp diff --git a/Core/GameEngineDevice/Include/Win32Device/Common/Win32BIGFileSystem.h b/Core/GameEngine/Include/Common/BIGFileSystem.h similarity index 91% rename from Core/GameEngineDevice/Include/Win32Device/Common/Win32BIGFileSystem.h rename to Core/GameEngine/Include/Common/BIGFileSystem.h index 6819ba90577..602a63e4dd5 100644 --- a/Core/GameEngineDevice/Include/Win32Device/Common/Win32BIGFileSystem.h +++ b/Core/GameEngine/Include/Common/BIGFileSystem.h @@ -22,7 +22,7 @@ // // //////////////////////////////////////////////////////////////////////////////// -//////// Win32BIGFileSystem.h /////////////////////////// +//////// BIGFileSystem.h /////////////////////////// // Bryan Cleveland, August 2002 ///////////////////////////////////////////////////////////// @@ -30,11 +30,11 @@ #include "Common/ArchiveFileSystem.h" -class Win32BIGFileSystem : public ArchiveFileSystem +class BIGFileSystem : public ArchiveFileSystem { public: - Win32BIGFileSystem(); - virtual ~Win32BIGFileSystem() override; + BIGFileSystem(); + virtual ~BIGFileSystem() override; virtual void init() override; virtual void update() override; diff --git a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp b/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp similarity index 82% rename from Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp rename to Core/GameEngine/Source/Common/System/BIGFileSystem.cpp index 2159c367728..6750525f15d 100644 --- a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp +++ b/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp @@ -22,13 +22,16 @@ // // //////////////////////////////////////////////////////////////////////////////// -//////// Win32BIGFileSystem.h /////////////////////////// +//////// BIGFileSystem.cpp /////////////////////////// // Bryan Cleveland, August 2002 ///////////////////////////////////////////////////////////// +#include "PreRTS.h" + +#include "Common/BIGFileSystem.h" + #include "Common/AudioAffect.h" #include "Common/ArchiveFile.h" -#include "Common/ArchiveFileSystem.h" #include "Common/file.h" #include "Common/GameAudio.h" #include "Common/GameMemory.h" @@ -38,19 +41,18 @@ #include "Common/Registry.h" #endif -#include "Win32Device/Common/Win32BIGFileSystem.h" #include "Utility/endian_compat.h" static const char *BIGFileIdentifier = "BIGF"; -Win32BIGFileSystem::Win32BIGFileSystem() : ArchiveFileSystem() { +BIGFileSystem::BIGFileSystem() : ArchiveFileSystem() { } -Win32BIGFileSystem::~Win32BIGFileSystem() { +BIGFileSystem::~BIGFileSystem() { } -void Win32BIGFileSystem::init() { +void BIGFileSystem::init() { DEBUG_ASSERTCRASH(TheLocalFileSystem != nullptr, ("TheLocalFileSystem must be initialized before TheArchiveFileSystem.")); if (TheLocalFileSystem == nullptr) { return; @@ -69,16 +71,16 @@ void Win32BIGFileSystem::init() { #endif } -void Win32BIGFileSystem::reset() { +void BIGFileSystem::reset() { } -void Win32BIGFileSystem::update() { +void BIGFileSystem::update() { } -void Win32BIGFileSystem::postProcessLoad() { +void BIGFileSystem::postProcessLoad() { } -ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { +ArchiveFile * BIGFileSystem::openArchiveFile(const Char *filename) { File *fp = TheLocalFileSystem->openFile(filename, File::READ | File::BINARY); AsciiString archiveFileName; archiveFileName = filename; @@ -86,7 +88,7 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { Int archiveFileSize = 0; Int numLittleFiles = 0; - DEBUG_LOG(("Win32BIGFileSystem::openArchiveFile - opening BIG file %s", filename)); + DEBUG_LOG(("BIGFileSystem::openArchiveFile - opening BIG file %s", filename)); if (fp == nullptr) { DEBUG_CRASH(("Could not open archive file %s for parsing", filename)); @@ -112,7 +114,7 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { return nullptr; } - DEBUG_LOG(("Win32BIGFileSystem::openArchiveFile - size of archive file is %d bytes", archiveFileSize)); + DEBUG_LOG(("BIGFileSystem::openArchiveFile - size of archive file is %d bytes", archiveFileSize)); // char t; @@ -126,7 +128,7 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { } numLittleFiles = betoh(numLittleFiles); - DEBUG_LOG(("Win32BIGFileSystem::openArchiveFile - %d are contained in archive", numLittleFiles)); + DEBUG_LOG(("BIGFileSystem::openArchiveFile - %d are contained in archive", numLittleFiles)); // for (Int i = 0; i < 2; ++i) { // t = buffer[i]; // buffer[i] = buffer[(4-i)-1]; @@ -187,7 +189,7 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { AsciiString debugpath; debugpath = path; debugpath.concat(fileInfo.m_filename); -// DEBUG_LOG(("Win32BIGFileSystem::openArchiveFile - adding file %s to archive file %s, file number %d", debugpath.str(), fileInfo->m_archiveFilename.str(), i)); +// DEBUG_LOG(("BIGFileSystem::openArchiveFile - adding file %s to archive file %s, file number %d", debugpath.str(), fileInfo->m_archiveFilename.str(), i)); archiveFile->addFile(path, &fileInfo); } @@ -207,7 +209,7 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { return archiveFile; } -void Win32BIGFileSystem::closeArchiveFile(const Char *filename) { +void BIGFileSystem::closeArchiveFile(const Char *filename) { // Need to close the specified big file ArchiveFileMap::iterator it = m_archiveFileMap.find(filename); if (it == m_archiveFileMap.end()) { @@ -228,13 +230,13 @@ void Win32BIGFileSystem::closeArchiveFile(const Char *filename) { m_archiveFileMap.erase(it); } -void Win32BIGFileSystem::closeAllArchiveFiles() { +void BIGFileSystem::closeAllArchiveFiles() { } -void Win32BIGFileSystem::closeAllFiles() { +void BIGFileSystem::closeAllFiles() { } -Bool Win32BIGFileSystem::loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite) { +Bool BIGFileSystem::loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite) { FilenameList filenameList; TheLocalFileSystem->getFileListInDirectory(dir, "", fileMask, filenameList, TRUE); @@ -255,10 +257,10 @@ Bool Win32BIGFileSystem::loadBigFilesFromDirectory(AsciiString dir, AsciiString ArchiveFile *archiveFile = openArchiveFile((*it).str()); if (archiveFile != nullptr) { - DEBUG_LOG(("Win32BIGFileSystem::loadBigFilesFromDirectory - loading %s into the directory tree.", (*it).str())); + DEBUG_LOG(("BIGFileSystem::loadBigFilesFromDirectory - loading %s into the directory tree.", (*it).str())); loadIntoDirectoryTree(archiveFile, overwrite); m_archiveFileMap[(*it)] = archiveFile; - DEBUG_LOG(("Win32BIGFileSystem::loadBigFilesFromDirectory - %s inserted into the archive file map.", (*it).str())); + DEBUG_LOG(("BIGFileSystem::loadBigFilesFromDirectory - %s inserted into the archive file map.", (*it).str())); actuallyAdded = TRUE; } diff --git a/Core/GameEngineDevice/CMakeLists.txt b/Core/GameEngineDevice/CMakeLists.txt index 3315487c387..71933e7c69f 100644 --- a/Core/GameEngineDevice/CMakeLists.txt +++ b/Core/GameEngineDevice/CMakeLists.txt @@ -83,7 +83,6 @@ set(GAMEENGINEDEVICE_SRC # Include/W3DDevice/GameLogic/W3DGameLogic.h # Include/W3DDevice/GameLogic/W3DGhostObject.h # Include/W3DDevice/GameLogic/W3DTerrainLogic.h - Include/Win32Device/Common/Win32BIGFileSystem.h # Include/Win32Device/Common/Win32GameEngine.h Include/Win32Device/Common/Win32LocalFile.h Include/Win32Device/Common/Win32LocalFileSystem.h @@ -185,7 +184,6 @@ set(GAMEENGINEDEVICE_SRC # Source/W3DDevice/GameLogic/W3DGameLogic.cpp # Source/W3DDevice/GameLogic/W3DGhostObject.cpp # Source/W3DDevice/GameLogic/W3DTerrainLogic.cpp - Source/Win32Device/Common/Win32BIGFileSystem.cpp # Source/Win32Device/Common/Win32GameEngine.cpp Source/Win32Device/Common/Win32LocalFile.cpp Source/Win32Device/Common/Win32LocalFileSystem.cpp @@ -198,10 +196,8 @@ set(GAMEENGINEDEVICE_SRC # Add C++ 17 FileSystem implementation for non-VS6 builds if(NOT IS_VS6_BUILD) list(APPEND GAMEENGINEDEVICE_SRC - Include/StdDevice/Common/StdBIGFileSystem.h Include/StdDevice/Common/StdLocalFile.h Include/StdDevice/Common/StdLocalFileSystem.h - Source/StdDevice/Common/StdBIGFileSystem.cpp Source/StdDevice/Common/StdLocalFile.cpp Source/StdDevice/Common/StdLocalFileSystem.cpp ) diff --git a/Core/GameEngineDevice/Include/StdDevice/Common/StdBIGFileSystem.h b/Core/GameEngineDevice/Include/StdDevice/Common/StdBIGFileSystem.h deleted file mode 100644 index b8cb316ab7c..00000000000 --- a/Core/GameEngineDevice/Include/StdDevice/Common/StdBIGFileSystem.h +++ /dev/null @@ -1,55 +0,0 @@ -/* -** Command & Conquer Generals Zero Hour(tm) -** Copyright 2025 Electronic Arts Inc. -** -** This program is free software: you can redistribute it and/or modify -** it under the terms of the GNU General Public License as published by -** the Free Software Foundation, either version 3 of the License, or -** (at your option) any later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program. If not, see . -*/ - -//////////////////////////////////////////////////////////////////////////////// -// // -// (c) 2001-2003 Electronic Arts Inc. // -// // -//////////////////////////////////////////////////////////////////////////////// - -//////// StdBIGFileSystem.h /////////////////////////// -// Stephan Vedder, April 2025 -///////////////////////////////////////////////////////////// - -#pragma once - -#include "Common/ArchiveFileSystem.h" - -class StdBIGFileSystem : public ArchiveFileSystem -{ -public: - StdBIGFileSystem(); - virtual ~StdBIGFileSystem() override; - - virtual void init() override; - virtual void update() override; - virtual void reset() override; - virtual void postProcessLoad() override; - - // ArchiveFile operations - virtual void closeAllArchiveFiles() override; ///< Close all Archivefiles currently open - - // File operations - virtual ArchiveFile * openArchiveFile(const Char *filename) override; - virtual void closeArchiveFile(const Char *filename) override; - virtual void closeAllFiles() override; ///< Close all files associated with ArchiveFiles - - virtual Bool loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) override; -protected: - -}; diff --git a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp b/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp deleted file mode 100644 index 529f5b03648..00000000000 --- a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp +++ /dev/null @@ -1,239 +0,0 @@ -/* -** Command & Conquer Generals Zero Hour(tm) -** Copyright 2025 Electronic Arts Inc. -** -** This program is free software: you can redistribute it and/or modify -** it under the terms of the GNU General Public License as published by -** the Free Software Foundation, either version 3 of the License, or -** (at your option) any later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program. If not, see . -*/ - -//////////////////////////////////////////////////////////////////////////////// -// // -// (c) 2001-2003 Electronic Arts Inc. // -// // -//////////////////////////////////////////////////////////////////////////////// - -//////// StdBIGFileSystem.h /////////////////////////// -// Stephan Vedder, April 2025 -///////////////////////////////////////////////////////////// - -#include "Common/AudioAffect.h" -#include "Common/ArchiveFile.h" -#include "Common/ArchiveFileSystem.h" -#include "Common/file.h" -#include "Common/GameAudio.h" -#include "Common/GameMemory.h" -#include "Common/LocalFileSystem.h" - -#if RTS_ZEROHOUR -#include "Common/Registry.h" -#endif - -#include "StdDevice/Common/StdBIGFileSystem.h" -#include "Utility/endian_compat.h" - -static const char *BIGFileIdentifier = "BIGF"; - -StdBIGFileSystem::StdBIGFileSystem() : ArchiveFileSystem() { -} - -StdBIGFileSystem::~StdBIGFileSystem() { -} - -void StdBIGFileSystem::init() { - DEBUG_ASSERTCRASH(TheLocalFileSystem != nullptr, ("TheLocalFileSystem must be initialized before TheArchiveFileSystem.")); - if (TheLocalFileSystem == nullptr) { - return; - } - - loadBigFilesFromDirectory("", "*.big"); - -#if RTS_ZEROHOUR - // load original Generals assets - AsciiString installPath; - GetStringFromGeneralsRegistry("", "InstallPath", installPath ); - //@todo this will need to be ramped up to a crash for release - DEBUG_ASSERTCRASH(!installPath.isEmpty(), ("Be 1337! Go install Generals!")); - if (!installPath.isEmpty()) - loadBigFilesFromDirectory(installPath, "*.big"); -#endif -} - -void StdBIGFileSystem::reset() { -} - -void StdBIGFileSystem::update() { -} - -void StdBIGFileSystem::postProcessLoad() { -} - -ArchiveFile * StdBIGFileSystem::openArchiveFile(const Char *filename) { - File *fp = TheLocalFileSystem->openFile(filename, File::READ | File::BINARY); - AsciiString archiveFileName; - archiveFileName = filename; - archiveFileName.toLower(); - Int archiveFileSize = 0; - Int numLittleFiles = 0; - - ArchiveFile *archiveFile = NEW StdBIGFile(filename, AsciiString::TheEmptyString); - - DEBUG_LOG(("StdBIGFileSystem::openArchiveFile - opening BIG file %s", filename)); - - if (fp == nullptr) { - DEBUG_CRASH(("Could not open archive file %s for parsing", filename)); - return nullptr; - } - - AsciiString asciibuf; - char buffer[_MAX_PATH]; - fp->read(buffer, 4); // read the "BIG" at the beginning of the file. - buffer[4] = 0; - if (strcmp(buffer, BIGFileIdentifier) != 0) { - DEBUG_CRASH(("Error reading BIG file identifier in file %s", filename)); - fp->close(); - fp = nullptr; - return nullptr; - } - - // read in the file size. - fp->read(&archiveFileSize, 4); - - DEBUG_LOG(("StdBIGFileSystem::openArchiveFile - size of archive file is %d bytes", archiveFileSize)); - -// char t; - - // read in the number of files contained in this BIG file. - // change the order of the bytes cause the file size is in reverse byte order for some reason. - fp->read(&numLittleFiles, 4); - numLittleFiles = betoh(numLittleFiles); - - DEBUG_LOG(("StdBIGFileSystem::openArchiveFile - %d are contained in archive", numLittleFiles)); -// for (Int i = 0; i < 2; ++i) { -// t = buffer[i]; -// buffer[i] = buffer[(4-i)-1]; -// buffer[(4-i)-1] = t; -// } - - // seek to the beginning of the directory listing. - fp->seek(0x10, File::START); - // read in each directory listing. - ArchivedFileInfo fileInfo; - ArchiveFile *archiveFile = NEW ArchiveFile(filename, AsciiString::TheEmptyString); - - for (Int i = 0; i < numLittleFiles; ++i) { - Int filesize = 0; - Int fileOffset = 0; - fp->read(&fileOffset, 4); - fp->read(&filesize, 4); - - filesize = betoh(filesize); - fileOffset = betoh(fileOffset); - - fileInfo.m_archiveFilename = archiveFileName; - fileInfo.m_offset = fileOffset; - fileInfo.m_size = filesize; - - // read in the path name of the file. - Int pathIndex = -1; - do { - ++pathIndex; - fp->read(buffer + pathIndex, 1); - } while (buffer[pathIndex] != 0); - - Int filenameIndex = pathIndex; - while ((filenameIndex >= 0) && (buffer[filenameIndex] != '\\') && (buffer[filenameIndex] != '/')) { - --filenameIndex; - } - - fileInfo.m_filename = (char *)(buffer + filenameIndex + 1); - fileInfo.m_filename.toLower(); - buffer[filenameIndex + 1] = 0; - - AsciiString path; - path = buffer; - - AsciiString debugpath; - debugpath = path; - debugpath.concat(fileInfo.m_filename); -// DEBUG_LOG(("StdBIGFileSystem::openArchiveFile - adding file %s to archive file %s, file number %d", debugpath.str(), fileInfo->m_archiveFilename.str(), i)); - - archiveFile->addFile(path, &fileInfo); - } - - archiveFile->attachFile(fp); - - // leave fp open as the archive file will be using it. - - return archiveFile; -} - -void StdBIGFileSystem::closeArchiveFile(const Char *filename) { - // Need to close the specified big file - ArchiveFileMap::iterator it = m_archiveFileMap.find(filename); - if (it == m_archiveFileMap.end()) { - return; - } - - if (stricmp(filename, MUSIC_BIG) == 0) { - // Stop the current audio - TheAudio->stopAudio(AudioAffect_Music); - - // No need to turn off other audio, as the lookups will just fail. - } - DEBUG_ASSERTCRASH(stricmp(filename, MUSIC_BIG) == 0, ("Attempting to close Archive file '%s', need to add code to handle its shutdown correctly.", filename)); - - // may need to do some other processing here first. - - delete (it->second); - m_archiveFileMap.erase(it); -} - -void StdBIGFileSystem::closeAllArchiveFiles() { -} - -void StdBIGFileSystem::closeAllFiles() { -} - -Bool StdBIGFileSystem::loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite) { - - FilenameList filenameList; - TheLocalFileSystem->getFileListInDirectory(dir, "", fileMask, filenameList, TRUE); - - Bool actuallyAdded = FALSE; - FilenameListIter it = filenameList.begin(); - while (it != filenameList.end()) { -#if RTS_ZEROHOUR - // TheSuperHackers @bugfix bobtista 18/11/2025 Skip duplicate INIZH.big in Data\INI to prevent CRC mismatches. - // English, Chinese, and Korean SKUs shipped with two INIZH.big files (one in Run directory, one in Run\Data\INI). - // The DeleteFile cleanup doesn't work on EA App/Origin installs because the folder is not writable, so we skip loading it instead. - if (it->endsWithNoCase("Data\\INI\\INIZH.big") || it->endsWithNoCase("Data/INI/INIZH.big")) { - it++; - continue; - } -#endif - - ArchiveFile *archiveFile = openArchiveFile((*it).str()); - - if (archiveFile != nullptr) { - DEBUG_LOG(("StdBIGFileSystem::loadBigFilesFromDirectory - loading %s into the directory tree.", (*it).str())); - loadIntoDirectoryTree(archiveFile, overwrite); - m_archiveFileMap[(*it)] = archiveFile; - DEBUG_LOG(("StdBIGFileSystem::loadBigFilesFromDirectory - %s inserted into the archive file map.", (*it).str())); - actuallyAdded = TRUE; - } - - it++; - } - - return actuallyAdded; -} diff --git a/Core/Tools/Autorun/autorun.cpp b/Core/Tools/Autorun/autorun.cpp index 2e5a428c2b3..4f3618dcf2a 100644 --- a/Core/Tools/Autorun/autorun.cpp +++ b/Core/Tools/Autorun/autorun.cpp @@ -119,7 +119,7 @@ #include "GameClient/GameText.h" #include "Common/UnicodeString.h" #include "Win32Device/Common/Win32LocalFileSystem.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" +#include "Common/BIGFileSystem.h" #endif //----------------------------------------------------------------------------- @@ -449,7 +449,7 @@ int PASCAL WinMain( HINSTANCE hInstance, HINSTANCE hPrevInstance, LPTSTR lpszCmd // create singletons TheFileSystem->init(); TheLocalFileSystem = new Win32LocalFileSystem; - TheArchiveFileSystem = new Win32BIGFileSystem; + TheArchiveFileSystem = new BIGFileSystem; TheGameText = CreateGameTextInterface(); TheGameText->init(); diff --git a/Core/Tools/MapCacheBuilder/Source/WinMain.cpp b/Core/Tools/MapCacheBuilder/Source/WinMain.cpp index ab545a07892..fe7fcf1bc5d 100644 --- a/Core/Tools/MapCacheBuilder/Source/WinMain.cpp +++ b/Core/Tools/MapCacheBuilder/Source/WinMain.cpp @@ -51,8 +51,8 @@ #include "Common/ThingFactory.h" #include "Common/FileSystem.h" +#include "Common/BIGFileSystem.h" #include "Win32Device/Common/Win32LocalFileSystem.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" #include "Common/SubsystemInterface.h" #include "GameClient/MapUtil.h" #include "W3DDevice/Common/W3DModuleFactory.h" @@ -101,7 +101,6 @@ #include #include "Win32Device/GameClient/Win32Mouse.h" #include "Win32Device/Common/Win32LocalFileSystem.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" #include "WWLib/trim.h" @@ -249,7 +248,7 @@ Int APIENTRY WinMain( HINSTANCE hInstance, HINSTANCE hPrevInstance, TheFileSystem = new FileSystem; initSubsystem(TheLocalFileSystem, (LocalFileSystem*)new Win32LocalFileSystem); - initSubsystem(TheArchiveFileSystem, (ArchiveFileSystem*)new Win32BIGFileSystem); + initSubsystem(TheArchiveFileSystem, (ArchiveFileSystem*)new BIGFileSystem); INI ini; initSubsystem(TheWritableGlobalData, new GlobalData(), "Data\\INI\\Default\\GameData.ini", "Data\\INI\\GameData.ini"); initSubsystem(TheGameText, CreateGameTextInterface()); diff --git a/Generals/Code/GameEngineDevice/CMakeLists.txt b/Generals/Code/GameEngineDevice/CMakeLists.txt index bedd7c48520..dbb41d0f7b4 100644 --- a/Generals/Code/GameEngineDevice/CMakeLists.txt +++ b/Generals/Code/GameEngineDevice/CMakeLists.txt @@ -70,7 +70,6 @@ set(GAMEENGINEDEVICE_SRC Include/W3DDevice/GameLogic/W3DGameLogic.h Include/W3DDevice/GameLogic/W3DGhostObject.h Include/W3DDevice/GameLogic/W3DTerrainLogic.h -# Include/Win32Device/Common/Win32BIGFileSystem.h Include/Win32Device/Common/Win32GameEngine.h # Include/Win32Device/Common/Win32LocalFile.h # Include/Win32Device/Common/Win32LocalFileSystem.h @@ -167,7 +166,6 @@ set(GAMEENGINEDEVICE_SRC Source/W3DDevice/GameLogic/W3DGameLogic.cpp Source/W3DDevice/GameLogic/W3DGhostObject.cpp Source/W3DDevice/GameLogic/W3DTerrainLogic.cpp -# Source/Win32Device/Common/Win32BIGFileSystem.cpp Source/Win32Device/Common/Win32GameEngine.cpp # Source/Win32Device/Common/Win32LocalFile.cpp # Source/Win32Device/Common/Win32LocalFileSystem.cpp diff --git a/Generals/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h b/Generals/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h index b17369dc844..01f5b0af14f 100644 --- a/Generals/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h +++ b/Generals/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h @@ -37,7 +37,7 @@ #include "GameLogic/GameLogic.h" #include "GameNetwork/NetworkInterface.h" #include "MilesAudioDevice/MilesAudioManager.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" +#include "Common/BIGFileSystem.h" #include "Win32Device/Common/Win32LocalFileSystem.h" #include "W3DDevice/Common/W3DModuleFactory.h" #include "W3DDevice/GameLogic/W3DGameLogic.h" @@ -93,7 +93,7 @@ inline ModuleFactory *Win32GameEngine::createModuleFactory() { return NEW W3DMod inline ThingFactory *Win32GameEngine::createThingFactory() { return NEW W3DThingFactory; } inline FunctionLexicon *Win32GameEngine::createFunctionLexicon() { return NEW W3DFunctionLexicon; } inline LocalFileSystem *Win32GameEngine::createLocalFileSystem() { return NEW Win32LocalFileSystem; } -inline ArchiveFileSystem *Win32GameEngine::createArchiveFileSystem() { return NEW Win32BIGFileSystem; } +inline ArchiveFileSystem *Win32GameEngine::createArchiveFileSystem() { return NEW BIGFileSystem; } inline ParticleSystemManager* Win32GameEngine::createParticleSystemManager(Bool dummy) { if (dummy) diff --git a/Generals/Code/Tools/GUIEdit/Source/GUIEdit.cpp b/Generals/Code/Tools/GUIEdit/Source/GUIEdit.cpp index 4d18b320cd8..d5c928ff52a 100644 --- a/Generals/Code/Tools/GUIEdit/Source/GUIEdit.cpp +++ b/Generals/Code/Tools/GUIEdit/Source/GUIEdit.cpp @@ -85,7 +85,7 @@ #include "Win32Device/GameClient/Win32Mouse.h" #include "Win32Device/GameClient/Win32DIKeyboard.h" #include "Win32Device/Common/Win32LocalFileSystem.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" +#include "Common/BIGFileSystem.h" #include "resource.h" #include "WinMain.h" @@ -485,7 +485,7 @@ void GUIEdit::init() // create file system TheFileSystem = new FileSystem; TheLocalFileSystem = new Win32LocalFileSystem; - TheArchiveFileSystem = new Win32BIGFileSystem; + TheArchiveFileSystem = new BIGFileSystem; TheFileSystem->init(); //--------------------------------------------------------------------------- diff --git a/Generals/Code/Tools/WorldBuilder/src/WorldBuilder.cpp b/Generals/Code/Tools/WorldBuilder/src/WorldBuilder.cpp index b3287235361..9c5421c320d 100644 --- a/Generals/Code/Tools/WorldBuilder/src/WorldBuilder.cpp +++ b/Generals/Code/Tools/WorldBuilder/src/WorldBuilder.cpp @@ -79,7 +79,7 @@ #include #include "Win32Device/GameClient/Win32Mouse.h" #include "Win32Device/Common/Win32LocalFileSystem.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" +#include "Common/BIGFileSystem.h" static SubsystemInterfaceList TheSubsystemListRecord; @@ -316,7 +316,7 @@ BOOL CWorldBuilderApp::InitInstance() TheFileSystem = new FileSystem; initSubsystem(TheLocalFileSystem, (LocalFileSystem*)new Win32LocalFileSystem); - initSubsystem(TheArchiveFileSystem, (ArchiveFileSystem*)new Win32BIGFileSystem); + initSubsystem(TheArchiveFileSystem, (ArchiveFileSystem*)new BIGFileSystem); // Just for kicks, get the HINSTANCE that WOL would need // if we were going to use it, which we aren't. diff --git a/GeneralsMD/Code/GameEngineDevice/CMakeLists.txt b/GeneralsMD/Code/GameEngineDevice/CMakeLists.txt index e8567307646..d251e04bb38 100644 --- a/GeneralsMD/Code/GameEngineDevice/CMakeLists.txt +++ b/GeneralsMD/Code/GameEngineDevice/CMakeLists.txt @@ -81,7 +81,6 @@ set(GAMEENGINEDEVICE_SRC Include/W3DDevice/GameLogic/W3DGameLogic.h Include/W3DDevice/GameLogic/W3DGhostObject.h Include/W3DDevice/GameLogic/W3DTerrainLogic.h -# Include/Win32Device/Common/Win32BIGFileSystem.h Include/Win32Device/Common/Win32GameEngine.h # Include/Win32Device/Common/Win32LocalFile.h # Include/Win32Device/Common/Win32LocalFileSystem.h @@ -180,7 +179,6 @@ set(GAMEENGINEDEVICE_SRC Source/W3DDevice/GameLogic/W3DGameLogic.cpp Source/W3DDevice/GameLogic/W3DGhostObject.cpp Source/W3DDevice/GameLogic/W3DTerrainLogic.cpp -# Source/Win32Device/Common/Win32BIGFileSystem.cpp Source/Win32Device/Common/Win32GameEngine.cpp # Source/Win32Device/Common/Win32LocalFile.cpp # Source/Win32Device/Common/Win32LocalFileSystem.cpp diff --git a/GeneralsMD/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h b/GeneralsMD/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h index 2710cdf4555..e27eb8504c9 100644 --- a/GeneralsMD/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h +++ b/GeneralsMD/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h @@ -32,12 +32,12 @@ #pragma once +#include "Common/BIGFileSystem.h" #include "Common/GameEngine.h" #include "GameClient/ParticleSys.h" #include "GameLogic/GameLogic.h" #include "GameNetwork/NetworkInterface.h" #include "MilesAudioDevice/MilesAudioManager.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" #include "Win32Device/Common/Win32LocalFileSystem.h" #include "W3DDevice/Common/W3DModuleFactory.h" #include "W3DDevice/GameLogic/W3DGameLogic.h" @@ -93,7 +93,7 @@ inline ModuleFactory *Win32GameEngine::createModuleFactory() { return NEW W3DMod inline ThingFactory *Win32GameEngine::createThingFactory() { return NEW W3DThingFactory; } inline FunctionLexicon *Win32GameEngine::createFunctionLexicon() { return NEW W3DFunctionLexicon; } inline LocalFileSystem *Win32GameEngine::createLocalFileSystem() { return NEW Win32LocalFileSystem; } -inline ArchiveFileSystem *Win32GameEngine::createArchiveFileSystem() { return NEW Win32BIGFileSystem; } +inline ArchiveFileSystem *Win32GameEngine::createArchiveFileSystem() { return NEW BIGFileSystem; } inline ParticleSystemManager* Win32GameEngine::createParticleSystemManager(Bool dummy) { if (dummy) diff --git a/GeneralsMD/Code/Tools/GUIEdit/Source/GUIEdit.cpp b/GeneralsMD/Code/Tools/GUIEdit/Source/GUIEdit.cpp index 2cbc8ec3ffb..6ecfad34952 100644 --- a/GeneralsMD/Code/Tools/GUIEdit/Source/GUIEdit.cpp +++ b/GeneralsMD/Code/Tools/GUIEdit/Source/GUIEdit.cpp @@ -85,7 +85,7 @@ #include "Win32Device/GameClient/Win32Mouse.h" #include "Win32Device/GameClient/Win32DIKeyboard.h" #include "Win32Device/Common/Win32LocalFileSystem.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" +#include "Common/BIGFileSystem.h" #include "resource.h" #include "WinMain.h" @@ -485,7 +485,7 @@ void GUIEdit::init() // create file system TheFileSystem = new FileSystem; TheLocalFileSystem = new Win32LocalFileSystem; - TheArchiveFileSystem = new Win32BIGFileSystem; + TheArchiveFileSystem = new BIGFileSystem; TheFileSystem->init(); //--------------------------------------------------------------------------- diff --git a/GeneralsMD/Code/Tools/WorldBuilder/src/WorldBuilder.cpp b/GeneralsMD/Code/Tools/WorldBuilder/src/WorldBuilder.cpp index d49589ee0c8..65d2fb82b8f 100644 --- a/GeneralsMD/Code/Tools/WorldBuilder/src/WorldBuilder.cpp +++ b/GeneralsMD/Code/Tools/WorldBuilder/src/WorldBuilder.cpp @@ -79,7 +79,7 @@ #include #include "Win32Device/GameClient/Win32Mouse.h" #include "Win32Device/Common/Win32LocalFileSystem.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" +#include "Common/BIGFileSystem.h" static SubsystemInterfaceList TheSubsystemListRecord; @@ -326,7 +326,7 @@ BOOL CWorldBuilderApp::InitInstance() TheFileSystem = new FileSystem; initSubsystem(TheLocalFileSystem, (LocalFileSystem*)new Win32LocalFileSystem); - initSubsystem(TheArchiveFileSystem, (ArchiveFileSystem*)new Win32BIGFileSystem); + initSubsystem(TheArchiveFileSystem, (ArchiveFileSystem*)new BIGFileSystem); // Just for kicks, get the HINSTANCE that WOL would need // if we were going to use it, which we aren't. From fcbc460c6861610084df4c5dff9f3a609f01289a Mon Sep 17 00:00:00 2001 From: xezon <4720891+xezon@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:11:49 +0200 Subject: [PATCH 09/17] refactor(archivefilesystem): Move loadMods to BIGFileSystem and rename loadBigFilesFromDirectory (#3505) --- .../Include/Common/ArchiveFileSystem.h | 4 +- .../GameEngine/Include/Common/BIGFileSystem.h | 4 +- .../Common/System/ArchiveFileSystem.cpp | 50 ---------------- .../Source/Common/System/BIGFileSystem.cpp | 60 +++++++++++++++++-- 4 files changed, 60 insertions(+), 58 deletions(-) diff --git a/Core/GameEngine/Include/Common/ArchiveFileSystem.h b/Core/GameEngine/Include/Common/ArchiveFileSystem.h index 5583290bce0..ddad61637a6 100644 --- a/Core/GameEngine/Include/Common/ArchiveFileSystem.h +++ b/Core/GameEngine/Include/Common/ArchiveFileSystem.h @@ -141,9 +141,9 @@ class ArchiveFileSystem : public SubsystemInterface void getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; ///< search the given directory for files matching the searchName (egs. *.ini, *.rep). Possibly search subdirectories. Scans each Archive file. Bool getFileInfo(const AsciiString& filename, FileInfo *fileInfo, FileInstance instance = 0) const; ///< see FileSystem.h - virtual Bool loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) = 0; + virtual Bool loadArchiveFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) = 0; - void loadMods(); + virtual void loadMods() = 0; ArchivedDirectoryInfo* friend_getArchivedDirectoryInfo(const Char* directory); diff --git a/Core/GameEngine/Include/Common/BIGFileSystem.h b/Core/GameEngine/Include/Common/BIGFileSystem.h index 602a63e4dd5..95e20c0ea6c 100644 --- a/Core/GameEngine/Include/Common/BIGFileSystem.h +++ b/Core/GameEngine/Include/Common/BIGFileSystem.h @@ -49,7 +49,9 @@ class BIGFileSystem : public ArchiveFileSystem virtual void closeArchiveFile(const Char *filename) override; virtual void closeAllFiles() override; ///< Close all files associated with ArchiveFiles - virtual Bool loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) override; + virtual Bool loadArchiveFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) override; + + virtual void loadMods() override; protected: }; diff --git a/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp b/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp index 17d49e8b372..2322827ef2b 100644 --- a/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp +++ b/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp @@ -51,8 +51,6 @@ #include "Common/AsciiString.h" #include "Common/LocalFileSystem.h" #include "Common/PerfTimer.h" -#include "../NGMP_include.h" -#include "../OnlineServices_Init.h" //---------------------------------------------------------------------------- @@ -231,54 +229,6 @@ void ArchiveFileSystem::loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool ove } } -void ArchiveFileSystem::loadMods() -{ -#if defined(GENERALS_ONLINE) && defined(GENERALS_ONLINE_COMMUNITY_PATCH_CHANGES) - // load community data patch BIG - if (NGMP_OnlineServicesManager::Settings.DataPacks_UseCommunityPatch() - && !TheGlobalData->m_commandLineData.isCommunityDataPatchDisabled()) - { - std::string strBigPath = std::format("{}GeneralsOnlineGameData/500_900_CommunityPatch_CoreINI.big", TheGlobalData->getPath_UserData().str()); - bool bLoaded = false; - if (TheLocalFileSystem->doesFileExist(strBigPath.c_str())) - { - ArchiveFile* archiveFile = openArchiveFile(strBigPath.c_str()); - if (archiveFile != nullptr) - { - // Sorted by filename so the patch respects the addon number order of BIGs in the game folder. - loadIntoDirectoryTree(archiveFile, FALSE, TRUE); - m_archiveFileMap[AsciiString(strBigPath.c_str())] = archiveFile; - bLoaded = true; - } - } - NetworkLog(ELogVerbosity::LOG_RELEASE, "Loaded community patch (%s): %d", strBigPath.c_str(), bLoaded); - } -#endif - - if (TheGlobalData->m_modBIG.isNotEmpty()) - { - ArchiveFile *archiveFile = openArchiveFile(TheGlobalData->m_modBIG.str()); - - if (archiveFile != nullptr) { - DEBUG_LOG(("ArchiveFileSystem::loadMods - loading %s into the directory tree.", TheGlobalData->m_modBIG.str())); - loadIntoDirectoryTree(archiveFile, TRUE); - m_archiveFileMap[TheGlobalData->m_modBIG] = archiveFile; - DEBUG_LOG(("ArchiveFileSystem::loadMods - %s inserted into the archive file map.", TheGlobalData->m_modBIG.str())); - } - else - { - DEBUG_LOG(("ArchiveFileSystem::loadMods - could not openArchiveFile(%s)", TheGlobalData->m_modBIG.str())); - } - } - - if (TheGlobalData->m_modDir.isNotEmpty()) - { - MAYBE_UNUSED Bool ret = loadBigFilesFromDirectory(TheGlobalData->m_modDir, "*.big", TRUE); - (void)ret; - DEBUG_ASSERTLOG(ret, ("loadBigFilesFromDirectory(%s) returned FALSE!", TheGlobalData->m_modDir.str())); - } -} - Bool ArchiveFileSystem::doesFileExist(const Char *filename, FileInstance instance) const { ArchivedDirectoryInfoResult result = const_cast(this)->getArchivedDirectoryInfo(filename); diff --git a/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp b/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp index 6750525f15d..3bd1a29026c 100644 --- a/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp +++ b/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp @@ -42,6 +42,8 @@ #endif #include "Utility/endian_compat.h" +#include "../NGMP_include.h" +#include "../OnlineServices_Init.h" static const char *BIGFileIdentifier = "BIGF"; @@ -58,7 +60,7 @@ void BIGFileSystem::init() { return; } - loadBigFilesFromDirectory("", "*.big"); + loadArchiveFilesFromDirectory("", "*.big"); #if RTS_ZEROHOUR // load original Generals assets @@ -67,7 +69,7 @@ void BIGFileSystem::init() { //@todo this will need to be ramped up to a crash for release DEBUG_ASSERTCRASH(!installPath.isEmpty(), ("Be 1337! Go install Generals!")); if (!installPath.isEmpty()) - loadBigFilesFromDirectory(installPath, "*.big"); + loadArchiveFilesFromDirectory(installPath, "*.big"); #endif } @@ -236,7 +238,7 @@ void BIGFileSystem::closeAllArchiveFiles() { void BIGFileSystem::closeAllFiles() { } -Bool BIGFileSystem::loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite) { +Bool BIGFileSystem::loadArchiveFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite) { FilenameList filenameList; TheLocalFileSystem->getFileListInDirectory(dir, "", fileMask, filenameList, TRUE); @@ -257,10 +259,10 @@ Bool BIGFileSystem::loadBigFilesFromDirectory(AsciiString dir, AsciiString fileM ArchiveFile *archiveFile = openArchiveFile((*it).str()); if (archiveFile != nullptr) { - DEBUG_LOG(("BIGFileSystem::loadBigFilesFromDirectory - loading %s into the directory tree.", (*it).str())); + DEBUG_LOG(("BIGFileSystem::loadArchiveFilesFromDirectory - loading %s into the directory tree.", (*it).str())); loadIntoDirectoryTree(archiveFile, overwrite); m_archiveFileMap[(*it)] = archiveFile; - DEBUG_LOG(("BIGFileSystem::loadBigFilesFromDirectory - %s inserted into the archive file map.", (*it).str())); + DEBUG_LOG(("BIGFileSystem::loadArchiveFilesFromDirectory - %s inserted into the archive file map.", (*it).str())); actuallyAdded = TRUE; } @@ -269,3 +271,51 @@ Bool BIGFileSystem::loadBigFilesFromDirectory(AsciiString dir, AsciiString fileM return actuallyAdded; } + +void BIGFileSystem::loadMods() +{ +#if defined(GENERALS_ONLINE) && defined(GENERALS_ONLINE_COMMUNITY_PATCH_CHANGES) + // load community data patch BIG + if (NGMP_OnlineServicesManager::Settings.DataPacks_UseCommunityPatch() + && !TheGlobalData->m_commandLineData.isCommunityDataPatchDisabled()) + { + std::string strBigPath = std::format("{}GeneralsOnlineGameData/500_900_CommunityPatch_CoreINI.big", TheGlobalData->getPath_UserData().str()); + bool bLoaded = false; + if (TheLocalFileSystem->doesFileExist(strBigPath.c_str())) + { + ArchiveFile* archiveFile = openArchiveFile(strBigPath.c_str()); + if (archiveFile != nullptr) + { + // Sorted by filename so the patch respects the addon number order of BIGs in the game folder. + loadIntoDirectoryTree(archiveFile, FALSE, TRUE); + m_archiveFileMap[AsciiString(strBigPath.c_str())] = archiveFile; + bLoaded = true; + } + } + NetworkLog(ELogVerbosity::LOG_RELEASE, "Loaded community patch (%s): %d", strBigPath.c_str(), bLoaded); + } +#endif + + if (TheGlobalData->m_modBIG.isNotEmpty()) + { + ArchiveFile *archiveFile = openArchiveFile(TheGlobalData->m_modBIG.str()); + + if (archiveFile != nullptr) { + DEBUG_LOG(("BIGFileSystem::loadMods - loading %s into the directory tree.", TheGlobalData->m_modBIG.str())); + loadIntoDirectoryTree(archiveFile, TRUE); + m_archiveFileMap[TheGlobalData->m_modBIG] = archiveFile; + DEBUG_LOG(("BIGFileSystem::loadMods - %s inserted into the archive file map.", TheGlobalData->m_modBIG.str())); + } + else + { + DEBUG_LOG(("BIGFileSystem::loadMods - could not openArchiveFile(%s)", TheGlobalData->m_modBIG.str())); + } + } + + if (TheGlobalData->m_modDir.isNotEmpty()) + { + MAYBE_UNUSED Bool ret = loadArchiveFilesFromDirectory(TheGlobalData->m_modDir, "*.big", TRUE); + (void)ret; + DEBUG_ASSERTLOG(ret, ("loadArchiveFilesFromDirectory(%s) returned FALSE!", TheGlobalData->m_modDir.str())); + } +} From d8cf0bb12e3ba2e993add7c6973a0bb0a58ab863 Mon Sep 17 00:00:00 2001 From: Tin Tin Hamans <5984296+tintinhamans@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:39:08 +0100 Subject: [PATCH 10/17] fix(bigfilesystem): Reject malformed BIG archives (#3588) --- .../Source/Common/System/BIGFileSystem.cpp | 94 +++++++++---------- 1 file changed, 45 insertions(+), 49 deletions(-) diff --git a/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp b/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp index 3bd1a29026c..d5fb1ec8fd6 100644 --- a/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp +++ b/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp @@ -46,7 +46,35 @@ #include "../OnlineServices_Init.h" -static const char *BIGFileIdentifier = "BIGF"; +static const char BIGFileIdentifier[] = { 'B', 'I', 'G', 'F' }; + +struct BIGFileHeader { + char identifier[sizeof(BIGFileIdentifier)]; + UnsignedInt archiveFileSize; // little endian, retail reads it as Int so 2 GB max + UnsignedInt numFiles; // big endian + UnsignedInt firstEntryOffset; // big endian +}; + +struct BIGFileEntryHeader { + UnsignedInt fileOffset; // big endian + UnsignedInt fileSize; // big endian +}; + +static_assert(sizeof(BIGFileHeader) == 16, "Incorrect BIG header size"); +static_assert(sizeof(BIGFileEntryHeader) == 8, "Incorrect BIG entry header size"); + +// Returns the length of the string, or -1 if it is truncated or does not fit the buffer. +static Int readNullTerminatedString(File *fp, char *buffer, Int bufferSize) { + for (Int i = 0; i < bufferSize; ++i) { + if (fp->read(buffer + i, 1) != 1) { + return -1; + } + if (buffer[i] == 0) { + return i; + } + } + return -1; +} BIGFileSystem::BIGFileSystem() : ArchiveFileSystem() { } @@ -87,8 +115,9 @@ ArchiveFile * BIGFileSystem::openArchiveFile(const Char *filename) { AsciiString archiveFileName; archiveFileName = filename; archiveFileName.toLower(); - Int archiveFileSize = 0; - Int numLittleFiles = 0; + UnsignedInt archiveFileSize = 0; + UnsignedInt numLittleFiles = 0; + BIGFileHeader header; DEBUG_LOG(("BIGFileSystem::openArchiveFile - opening BIG file %s", filename)); @@ -99,79 +128,46 @@ ArchiveFile * BIGFileSystem::openArchiveFile(const Char *filename) { AsciiString asciibuf; char buffer[_MAX_PATH]; - // read the "BIG" at the beginning of the file. - buffer[4] = 0; - if (fp->read(buffer, 4) != 4 || strcmp(buffer, BIGFileIdentifier) != 0) { - DEBUG_CRASH(("Error reading BIG file identifier in file %s", filename)); + if (fp->read(&header, sizeof(header)) != sizeof(header) || memcmp(header.identifier, BIGFileIdentifier, sizeof(BIGFileIdentifier)) != 0) { + DEBUG_CRASH(("Error reading BIG file header in file %s", filename)); fp->close(); fp = nullptr; return nullptr; } - // read in the file size. - if (fp->read(&archiveFileSize, 4) != 4) { - DEBUG_CRASH(("Error reading BIG file size in file %s", filename)); - fp->close(); - fp = nullptr; - return nullptr; - } + archiveFileSize = letoh(header.archiveFileSize); + numLittleFiles = betoh(header.numFiles); - DEBUG_LOG(("BIGFileSystem::openArchiveFile - size of archive file is %d bytes", archiveFileSize)); + DEBUG_LOG(("BIGFileSystem::openArchiveFile - size of archive file is %u bytes", archiveFileSize)); // char t; - // read in the number of files contained in this BIG file. - // change the order of the bytes cause the file size is in reverse byte order for some reason. - if (fp->read(&numLittleFiles, 4) != 4) { - DEBUG_CRASH(("Error reading BIG file count in file %s", filename)); - fp->close(); - fp = nullptr; - return nullptr; - } - numLittleFiles = betoh(numLittleFiles); - - DEBUG_LOG(("BIGFileSystem::openArchiveFile - %d are contained in archive", numLittleFiles)); + DEBUG_LOG(("BIGFileSystem::openArchiveFile - %u are contained in archive", numLittleFiles)); // for (Int i = 0; i < 2; ++i) { // t = buffer[i]; // buffer[i] = buffer[(4-i)-1]; // buffer[(4-i)-1] = t; // } - // seek to the beginning of the directory listing. - fp->seek(0x10, File::START); // read in each directory listing. ArchivedFileInfo fileInfo; ArchiveFile *archiveFile = NEW ArchiveFile(filename, AsciiString::TheEmptyString); Bool corrupt = FALSE; - for (Int i = 0; i < numLittleFiles; ++i) { - Int filesize = 0; - Int fileOffset = 0; - if (fp->read(&fileOffset, 4) != 4 || fp->read(&filesize, 4) != 4) { + for (UnsignedInt i = 0; i < numLittleFiles; ++i) { + BIGFileEntryHeader entry; + if (fp->read(&entry, sizeof(entry)) != sizeof(entry)) { corrupt = TRUE; break; } - filesize = betoh(filesize); - fileOffset = betoh(fileOffset); - fileInfo.m_archiveFilename = archiveFileName; - fileInfo.m_offset = fileOffset; - fileInfo.m_size = filesize; + fileInfo.m_offset = betoh(entry.fileOffset); + fileInfo.m_size = betoh(entry.fileSize); // read in the path name of the file. - // the name must terminate within the buffer - Int pathIndex = -1; - Bool terminated = FALSE; - while (++pathIndex < (Int)sizeof(buffer)) { - if (fp->read(buffer + pathIndex, 1) != 1) - break; - if (buffer[pathIndex] == 0) { - terminated = TRUE; - break; - } - } - if (!terminated) { + const Int pathIndex = readNullTerminatedString(fp, buffer, sizeof(buffer)); + if (pathIndex < 0) { corrupt = TRUE; break; } From 42596c256a32436e84fb0e8990b8519cdee2a372 Mon Sep 17 00:00:00 2001 From: Okladnoj Date: Thu, 22 Oct 2026 20:11:04 +0300 Subject: [PATCH 11/17] bugfix(filesystem): Use lower_bound instead of find to place an overwriting archive first --- Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp b/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp index 2322827ef2b..44c436c9227 100644 --- a/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp +++ b/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp @@ -177,7 +177,7 @@ void ArchiveFileSystem::loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool ove else if (overwrite) { // When overwriting, try place the new value at the beginning of the key list. - fileIt = dirInfo->m_files.find(token); + fileIt = dirInfo->m_files.lower_bound(token); } else { From 2fb01fd6545a5954720dad10982eaf1523613da8 Mon Sep 17 00:00:00 2001 From: xezon <4720891+xezon@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:49:17 +0200 Subject: [PATCH 12/17] fix(string): Fix token clearing in AsciiString::nextToken(), UnicodeString::nextToken() (#3374) --- Core/GameEngine/Include/Common/AsciiString.h | 4 ++-- Core/GameEngine/Include/Common/UnicodeString.h | 4 ++-- .../Source/Common/System/AsciiString.cpp | 17 +++++++++++------ .../Source/Common/System/UnicodeString.cpp | 17 +++++++++++------ .../GameEngine/Source/Common/CommandLine.cpp | 1 + .../GameEngine/Source/Common/CommandLine.cpp | 1 + 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/Core/GameEngine/Include/Common/AsciiString.h b/Core/GameEngine/Include/Common/AsciiString.h index 66e4982927e..759f72ca913 100644 --- a/Core/GameEngine/Include/Common/AsciiString.h +++ b/Core/GameEngine/Include/Common/AsciiString.h @@ -353,12 +353,12 @@ class AsciiString /** conceptually similar to strtok(): - extract the next seps-delimited token from the front + extract the next separators-delimited token from the front of 'this' and copy it into 'token', returning true if a nonempty token was found. (note that this modifies 'this' as well, stripping the token off!) */ - Bool nextToken(AsciiString* token, const char* seps = nullptr); + Bool nextToken(AsciiString* token, const char* separators = nullptr); /** return true iff the string is "NONE" (case-insensitive). diff --git a/Core/GameEngine/Include/Common/UnicodeString.h b/Core/GameEngine/Include/Common/UnicodeString.h index 0f3c2333c44..58d73d183ea 100644 --- a/Core/GameEngine/Include/Common/UnicodeString.h +++ b/Core/GameEngine/Include/Common/UnicodeString.h @@ -343,12 +343,12 @@ class UnicodeString /** conceptually similar to strtok(): - extract the next whitespace-delimited token from the front + extract the next separators-delimited token from the front of 'this' and copy it into 'token', returning true if a nonempty token was found. (note that this modifies 'this' as well, stripping the token off!) */ - Bool nextToken(UnicodeString* token, UnicodeString delimiters = UnicodeString::TheEmptyString); + Bool nextToken(UnicodeString* token, const WideChar* separators = nullptr); // // You might think it would be a good idea to overload the * operator diff --git a/Core/GameEngine/Source/Common/System/AsciiString.cpp b/Core/GameEngine/Source/Common/System/AsciiString.cpp index afa1b96186b..1cb3bfb6046 100644 --- a/Core/GameEngine/Source/Common/System/AsciiString.cpp +++ b/Core/GameEngine/Source/Common/System/AsciiString.cpp @@ -607,16 +607,21 @@ Bool AsciiString::isNone() const } //----------------------------------------------------------------------------- -Bool AsciiString::nextToken(AsciiString* tok, const char* seps) +Bool AsciiString::nextToken(AsciiString* tok, const char* separators) { - if (this->isEmpty() || tok == this) + DEBUG_ASSERTCRASH(tok != this, ("Tokenizer and Token cannot be the same object")); + + if (this->isEmpty()) + { + tok->clear(); return false; + } - if (seps == nullptr) - seps = " \n\r\t"; + if (separators == nullptr) + separators = " \n\r\t"; - char* start = skipSeps(peek(), seps); - char* end = skipNonSeps(start, seps); + char* start = skipSeps(peek(), separators); + char* end = skipNonSeps(start, separators); if (end > start) { diff --git a/Core/GameEngine/Source/Common/System/UnicodeString.cpp b/Core/GameEngine/Source/Common/System/UnicodeString.cpp index e21c0ea38dc..3e8980b6395 100644 --- a/Core/GameEngine/Source/Common/System/UnicodeString.cpp +++ b/Core/GameEngine/Source/Common/System/UnicodeString.cpp @@ -460,20 +460,25 @@ Bool UnicodeString::endsWithNoCase(const WideChar* p) const } //----------------------------------------------------------------------------- -Bool UnicodeString::nextToken(UnicodeString* tok, UnicodeString delimiters) +Bool UnicodeString::nextToken(UnicodeString* tok, const WideChar* separators) { - if (this->isEmpty() || tok == this) + DEBUG_ASSERTCRASH(tok != this, ("Tokenizer and Token cannot be the same object")); + + if (this->isEmpty()) + { + tok->clear(); return false; + } - if (delimiters.isEmpty()) - delimiters = L" \t\n\r"; + if (separators == nullptr) + separators = L" \t\n\r"; Int offset; - offset = wcsspn(peek(), delimiters.str()); + offset = wcsspn(peek(), separators); WideChar* start = peek() + offset; - offset = wcscspn(start, delimiters.str()); + offset = wcscspn(start, separators); WideChar* end = start + offset; if (end > start) diff --git a/Generals/Code/GameEngine/Source/Common/CommandLine.cpp b/Generals/Code/GameEngine/Source/Common/CommandLine.cpp index 772830f0f67..0eee5949dc0 100644 --- a/Generals/Code/GameEngine/Source/Common/CommandLine.cpp +++ b/Generals/Code/GameEngine/Source/Common/CommandLine.cpp @@ -93,6 +93,7 @@ static void ConvertShortMapPathToLongMapPath(AsciiString &mapName) if (!token.endsWithNoCase(".map")) { DEBUG_CRASH(("Invalid map name %s", mapName.str())); + return; } // remove the .map from the end. token.truncateBy(4); diff --git a/GeneralsMD/Code/GameEngine/Source/Common/CommandLine.cpp b/GeneralsMD/Code/GameEngine/Source/Common/CommandLine.cpp index 7a43b85c8ca..3e1f2f91917 100644 --- a/GeneralsMD/Code/GameEngine/Source/Common/CommandLine.cpp +++ b/GeneralsMD/Code/GameEngine/Source/Common/CommandLine.cpp @@ -93,6 +93,7 @@ static void ConvertShortMapPathToLongMapPath(AsciiString &mapName) if (!token.endsWithNoCase(".map")) { DEBUG_CRASH(("Invalid map name %s", mapName.str())); + return; } // remove the .map from the end. token.truncateBy(4); From 24e01a76a486ad902835a8cb67bc04d1060ce26e Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:43:44 +0200 Subject: [PATCH 13/17] fix(filesystem): Only list archive subdirectories when asked --- .../Source/Common/System/ArchiveFile.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp index 9e0db3f0ee2..ce82fa0b4a8 100644 --- a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp +++ b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp @@ -150,15 +150,17 @@ void ArchiveFile::getFileListInDirectory(const AsciiString& currentDirectory, co void ArchiveFile::getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const { - DetailedArchivedDirectoryInfoMap::const_iterator diriter = dirInfo->m_directories.begin(); - for (; diriter != dirInfo->m_directories.end(); ++diriter) { - const DetailedArchivedDirectoryInfo *tempDirInfo = &(diriter->second); - AsciiString tempDirName = currentDirectory; - if (!tempDirName.isEmpty() && !tempDirName.endsWith("\\")) { - tempDirName.concat('\\'); + if (searchSubdirectories) { + DetailedArchivedDirectoryInfoMap::const_iterator diriter = dirInfo->m_directories.begin(); + for (; diriter != dirInfo->m_directories.end(); ++diriter) { + const DetailedArchivedDirectoryInfo *tempDirInfo = &(diriter->second); + AsciiString tempDirName = currentDirectory; + if (!tempDirName.isEmpty() && !tempDirName.endsWith("\\")) { + tempDirName.concat('\\'); + } + tempDirName.concat(tempDirInfo->m_directoryName); + getFileListInDirectory(tempDirInfo, tempDirName, searchName, filenameList, searchSubdirectories); } - tempDirName.concat(tempDirInfo->m_directoryName); - getFileListInDirectory(tempDirInfo, tempDirName, searchName, filenameList, searchSubdirectories); } ArchivedFileInfoMap::const_iterator fileiter = dirInfo->m_files.begin(); From 693c371d4c82c2dc067e76810dde4a2969b9c280 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Sat, 24 Oct 2026 14:21:23 +0200 Subject: [PATCH 14/17] feat(online): Mount archives by priority and read the data pack layers from settings --- Core/GameEngine/Include/Common/ArchiveFile.h | 3 + .../Include/Common/ArchiveFileSystem.h | 51 ++- Core/GameEngine/Include/Common/FileSystem.h | 35 ++ .../Common/System/ArchiveFileSystem.cpp | 354 +++++++++++++++--- .../Source/Common/System/BIGFileSystem.cpp | 61 +-- .../Source/Common/System/FileSystem.cpp | 206 +++++++++- .../GeneralsOnline/GeneralsOnline_Settings.h | 18 +- .../GameEngine/Source/Common/GameEngine.cpp | 7 +- .../GUI/GameWindowManagerScript.cpp | 21 -- .../GeneralsOnline_Settings.cpp | 109 +++++- scripts/pack_go_data.py | 63 ++++ 11 files changed, 815 insertions(+), 113 deletions(-) create mode 100644 scripts/pack_go_data.py diff --git a/Core/GameEngine/Include/Common/ArchiveFile.h b/Core/GameEngine/Include/Common/ArchiveFile.h index a7e0ed68bfb..61a43117017 100644 --- a/Core/GameEngine/Include/Common/ArchiveFile.h +++ b/Core/GameEngine/Include/Common/ArchiveFile.h @@ -58,6 +58,8 @@ class ArchiveFile void setSearchPriority( Int new_priority ); ///< Set this archive file's search priority void close(); ///< Close this archive file void attachFile(File *file); + void setPriority(Int priority) { m_priority = priority; } + Int getPriority() const { return m_priority; } void getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; @@ -68,6 +70,7 @@ class ArchiveFile const ArchivedFileInfo * getArchivedFileInfo(const AsciiString& filename) const; ///< return the ArchivedFileInfo from the directory tree. protected: + Int m_priority = ARCHIVE_PRIORITY_RETAIL; ///< lookup priority in the archive file system, lower wins File *m_file; ///< file pointer to the archive file on disk. Kept open so we don't have to continuously open and close the file all the time. FileInfo m_fileInfo; ///< file info of the archive file on disk. AsciiString m_name; ///< archive file name diff --git a/Core/GameEngine/Include/Common/ArchiveFileSystem.h b/Core/GameEngine/Include/Common/ArchiveFileSystem.h index ddad61637a6..381765359ac 100644 --- a/Core/GameEngine/Include/Common/ArchiveFileSystem.h +++ b/Core/GameEngine/Include/Common/ArchiveFileSystem.h @@ -53,6 +53,8 @@ #include "Common/AsciiString.h" #include "Common/FileSystem.h" // for typedefs, etc. #include "Common/STLTypedefs.h" +#include +#include //---------------------------------------------------------------------------- // Forward References @@ -120,6 +122,31 @@ class ArchivedFileInfo }; +// Lookup priority of an archive, lower wins. The archive mounted first wins on equal priority. +enum ArchivePriority +{ + ARCHIVE_PRIORITY_EXCLUDED = -1, ///< not mounted: a BIG of a game folder that matches user.exclude_paths + ARCHIVE_PRIORITY_MOD = 0, ///< -mod and -modDir, 0 to 999, a later one is lower and wins + ARCHIVE_PRIORITY_GAME_FOLDER = 1000, ///< BIGs of the Zero Hour folder that are not retail, in name order + ARCHIVE_PRIORITY_GAME_FOLDER_GENERALS = 1100, ///< BIGs of the Generals data (ZH_Generals or the Generals install) that are not retail + ARCHIVE_PRIORITY_USER = 100000, ///< data_packs.user of settings.json, 100000 plus the position in its include_paths + ARCHIVE_PRIORITY_GENERALS_ONLINE = 1000000, ///< data_packs.generals_online, 1000000 plus the position in its include_paths + ARCHIVE_PRIORITY_COMMUNITY_PATCH = 2000000, ///< data_packs.community_patch, 2000000 plus the position in its include_paths + ARCHIVE_PRIORITY_RETAIL = 3000000, ///< Zero Hour retail and official patch BIGs + ARCHIVE_PRIORITY_RETAIL_GENERALS = 3000100 ///< Generals retail and official patch BIGs +}; + +// One entry of the include_paths of a data pack layer in settings.json, see ArchiveFileSystem::parseMountEntry +struct ArchiveMountEntry +{ + enum Kind { KIND_BIG, KIND_BIG_GLOB }; + enum Base { BASE_GAME, BASE_USER, BASE_ABSOLUTE }; + + Kind m_kind; + Base m_base; + AsciiString m_path; ///< with backslashes, relative to the base, the BASE_ABSOLUTE path is complete +}; + class ArchiveFileSystem : public SubsystemInterface { public: @@ -145,6 +172,19 @@ class ArchiveFileSystem : public SubsystemInterface virtual void loadMods() = 0; + /// Parses an entry of the include_paths of a data pack layer: "game:", "user:" or an absolute path. A path ending in .big is one BIG, + /// a last component with * or ? and ending in .big is a set of BIGs, a path ending in a slash is every BIG in that folder. + /// Returns FALSE and a reason for an entry that is not valid. + static Bool parseMountEntry(const AsciiString& text, ArchiveMountEntry* entry, AsciiString* reason); + + /// TRUE for the retail archives and the official patches of Zero Hour and Generals, in every language. + static Bool isGeneralsArchivePath(const AsciiString& path, Bool generalsFolder); ///< TRUE for the Generals archives: the Generals install folder, or a ZH_Generals folder + + static Bool isRetailArchive(const AsciiString& path, Bool generalsFolder, Bool zeroHour); + + /// BIGs of the game folders that are not retail and not excluded by user.exclude_paths, which are mounted above the data pack layers + const std::vector& getGameFolderArchives() const { return m_gameFolderArchives; } + ArchivedDirectoryInfo* friend_getArchivedDirectoryInfo(const Char* directory); protected: @@ -160,9 +200,18 @@ class ArchiveFileSystem : public SubsystemInterface ArchiveFile* getArchiveFile(const AsciiString& filename, FileInstance instance = 0) const; ArchivedDirectoryInfoResult getArchivedDirectoryInfo(const Char* directory); - virtual void loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool overwrite = FALSE, Bool sortedByName = FALSE); ///< load the archive file's header information and apply it to the global archive directory tree. sortedByName inserts by archive filename order (addon number convention), ignoring overwrite. + virtual void loadIntoDirectoryTree(ArchiveFile *archiveFile, Int priority = ARCHIVE_PRIORITY_RETAIL); ///< load the archive file's header information and apply it to the global archive directory tree. The lowest priority number wins, the first mounted wins on equal priority. + + Int getNextModPriority() { return m_nextModPriority > ARCHIVE_PRIORITY_MOD ? m_nextModPriority-- : ARCHIVE_PRIORITY_MOD; } ///< -mod and -modDir: a later one wins + Int getGameFolderArchivePriority(const AsciiString& path, Bool generalsFolder, Bool zeroHour); ///< retail archives are the bottom, any other BIG of the game folders ranks above the data pack layers unless user.exclude_paths matches it + + void mountListedArchive(const ArchiveMountEntry& entry, Int priority, const PathExcludeList* excludes); + void mountDataPackLayer(const char* layer, Bool enabled, const std::vector& includePaths, const PathExcludeList* excludes, Int priority); ///< mounts the include_paths of one layer in list order, the first one wins, minus what excludes matches ArchiveFileMap m_archiveFileMap; + std::vector m_gameFolderArchives; + AsciiString m_generalsBaseSource; ///< where the Generals data came from, for the log + Int m_nextModPriority = ARCHIVE_PRIORITY_MOD + 999; ArchivedDirectoryInfo m_rootDirectory; }; diff --git a/Core/GameEngine/Include/Common/FileSystem.h b/Core/GameEngine/Include/Common/FileSystem.h index 75f93caabb5..d4a518434d6 100644 --- a/Core/GameEngine/Include/Common/FileSystem.h +++ b/Core/GameEngine/Include/Common/FileSystem.h @@ -53,6 +53,9 @@ #include +#include +#include + #include "WWLib/mutex.h" //---------------------------------------------------------------------------- @@ -126,6 +129,30 @@ struct FileInfo { Int timestampLow; }; +//=============================== +// PathExcludeList +//=============================== +/** + * The exclude_paths of settings.json. An entry is game:, user: or an absolute pattern, where * matches any text + * including folders and ? one character. A pattern ending in a slash is a folder and everything in it. Matching ignores case and slash direction. + * A path is tested as a key: game:, user: or the absolute path. + */ +class PathExcludeList +{ +public: + static Bool parse(const std::string& text, std::string* pattern, AsciiString* reason); ///< FALSE and a reason for an entry that is not valid + static std::string makeKey(const Char* path); ///< key of a path that is relative to the game folder or absolute + static std::string makeKey(const Char* path, const Char* userDataPath); ///< same, a path inside the user data folder becomes user: + + void set(const std::vector& entries); ///< entries that are not valid are dropped + Bool isEmpty() const { return m_patterns.empty(); } + Bool matches(const std::string& key) const; + Bool matchesPath(const Char* path) const { return !m_patterns.empty() && matches(makeKey(path)); } + +private: + std::vector m_patterns; +}; + //=============================== // FileSystem //=============================== @@ -165,6 +192,10 @@ class FileSystem : public SubsystemInterface Bool createDirectory(AsciiString directory); ///< create a directory of the given name. + /// Loose files of the game folder that match are treated as if they did not exist: not opened, not found, not listed. Files that are written and files outside the game folder are not affected. + void setExcludedPaths(const std::vector& entries) { m_excludedPaths.set(entries); } + const PathExcludeList& getExcludedPaths() const { return m_excludedPaths; } + static AsciiString normalizePath(const AsciiString& path); ///< normalizes a file path. The path can refer to a directory. File path must be absolute, but does not need to exist. Returns an empty string on failure. static Bool isPathInDirectory(const AsciiString& testPath, const AsciiString& basePath); ///< determines if a file path is within a base path. Both paths must be absolute, but do not need to exist. @@ -172,6 +203,10 @@ class FileSystem : public SubsystemInterface static bool removeExtension(UnicodeString& path); protected: + Bool isLooseFileExcluded(const Char *filename) const { return m_excludedPaths.matchesPath(filename); } + + PathExcludeList m_excludedPaths; + #if ENABLE_FILESYSTEM_EXISTENCE_CACHE struct FileExistData { diff --git a/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp b/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp index 44c436c9227..6171f26fabe 100644 --- a/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp +++ b/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp @@ -46,11 +46,18 @@ //---------------------------------------------------------------------------- #include "PreRTS.h" +#include +#include +#include +#include #include "Common/ArchiveFile.h" #include "Common/ArchiveFileSystem.h" #include "Common/AsciiString.h" #include "Common/LocalFileSystem.h" #include "Common/PerfTimer.h" +#include "Common/Registry.h" +#include "../NGMP_include.h" +#include "../OnlineServices_Init.h" //---------------------------------------------------------------------------- @@ -125,8 +132,9 @@ ArchiveFileSystem::~ArchiveFileSystem() } } -void ArchiveFileSystem::loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool overwrite, Bool sortedByName) +void ArchiveFileSystem::loadIntoDirectoryTree(ArchiveFile *archiveFile, Int priority) { + archiveFile->setPriority(priority); FilenameList filenameList; @@ -164,68 +172,308 @@ void ArchiveFileSystem::loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool ove tokenizer.nextToken(&token, "\\/"); } - ArchivedFileLocationMap::iterator fileIt; - if (sortedByName) - { - // Insert by case-insensitive archive filename, matching game folder load order where the alphabetically first archive wins. - const AsciiString baseName = getBaseFilename(archiveFile->getName()); - std::pair range = dirInfo->m_files.equal_range(token); - fileIt = range.first; - while (fileIt != range.second && getBaseFilename(fileIt->second->getName()).compareNoCase(baseName) <= 0) - ++fileIt; - } - else if (overwrite) + // Insert behind every archive of the same or a lower priority number, so the first one mounted wins on equal priority. + // A plain insert goes to the end of the range of equal keys, the range is rebuilt when the new archive belongs in front of others. + std::pair range = dirInfo->m_files.equal_range(token); + std::vector ordered; + for (ArchivedFileLocationMap::iterator rangeIt = range.first; rangeIt != range.second; ++rangeIt) + ordered.push_back(rangeIt->second); + + size_t position = 0; + while (position < ordered.size() && ordered[position]->getPriority() <= priority) + ++position; + + if (position == ordered.size()) { - // When overwriting, try place the new value at the beginning of the key list. - fileIt = dirInfo->m_files.lower_bound(token); + dirInfo->m_files.insert(std::make_pair(token, archiveFile)); } else { - // Append to the end of the key list. - fileIt = dirInfo->m_files.end(); + ordered.insert(ordered.begin() + position, archiveFile); + dirInfo->m_files.erase(range.first, range.second); + for (size_t k = 0; k < ordered.size(); ++k) + dirInfo->m_files.insert(std::make_pair(token, ordered[k])); } - dirInfo->m_files.insert(fileIt, std::make_pair(token, archiveFile)); + DEBUG_LOG(("ArchiveFileSystem::loadIntoDirectoryTree - adding file %s, archived in %s, priority %d", it->str(), archiveFile->getName().str(), priority)); + + it++; + } +} + +// Retail archive names, without the extension and in lower case. Only the language of the game counts as retail. +static void addRetailNames(std::set& names, Bool zeroHour) +{ + auto add = [&names](const std::string& name) + { + std::string lower = name; + for (char& c : lower) + c = static_cast(tolower(static_cast(c))); + names.insert(lower); + }; + + const std::string language = GetRegistryLanguage().str(); + + if (zeroHour) + { + add(language + "ZH"); + add("Audio" + language + "ZH"); + add("Speech" + language + "ZH"); + add("W3D" + language + "ZH"); + for (const char* name : { "AudioZH", "INIZH", "MapsZH", "Music", "MusicZH", "ShadersZH", "SpeechZH", "TerrainZH", "TexturesZH", "W3DZH", "WindowZH", "gensecZH", + "PatchData", "PatchINI", "PatchWindow", "PatchZH" }) + add(name); + } + else + { + add(language); + add("Audio" + language); + add("Speech" + language); + for (const char* name : { "Audio", "INI", "Music", "Patch", "PatchData", "PatchINI", "PatchWindow", "Speech", "Terrain", "Textures", "W3D", "Window", "gensec", "maps", "shaders" }) + add(name); + } +} + +Bool ArchiveFileSystem::isGeneralsArchivePath(const AsciiString& path, Bool generalsFolder) +{ + if (generalsFolder) + return TRUE; + + AsciiString lowerPath = path; + lowerPath.toLower(); + return strstr(lowerPath.str(), "zh_generals\\") != nullptr || strstr(lowerPath.str(), "zh_generals/") != nullptr; +} + +Bool ArchiveFileSystem::isRetailArchive(const AsciiString& path, Bool generalsFolder, Bool zeroHour) +{ + static std::set zeroHourNames; + static std::set generalsNames; + if (zeroHourNames.empty()) + { + addRetailNames(zeroHourNames, TRUE); + addRetailNames(generalsNames, FALSE); + } + + AsciiString name = getBaseFilename(path); + if (!name.endsWithNoCase(".big")) + return FALSE; + name.truncateBy(4); + std::string lower = name.str(); + for (char& c : lower) + c = static_cast(tolower(static_cast(c))); + + // The Generals archives are in the Generals install folder, or in a ZH_Generals folder inside the game folder + const Bool inGeneralsFolder = isGeneralsArchivePath(path, generalsFolder); + + if (zeroHour && !inGeneralsFolder) + return zeroHourNames.count(lower) != 0; + return generalsNames.count(lower) != 0; +} + +Int ArchiveFileSystem::getGameFolderArchivePriority(const AsciiString& path, Bool generalsFolder, Bool zeroHour) +{ + // The Zero Hour archives rank above the Generals ones in their tier, whatever the order of the scan + const Bool generals = zeroHour && isGeneralsArchivePath(path, generalsFolder); + if (isRetailArchive(path, generalsFolder, zeroHour)) + return generals ? ARCHIVE_PRIORITY_RETAIL_GENERALS : ARCHIVE_PRIORITY_RETAIL; + + if (TheFileSystem != nullptr && TheFileSystem->getExcludedPaths().matchesPath(path.str())) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Archive %s is excluded by user.exclude_paths", path.str()); + return ARCHIVE_PRIORITY_EXCLUDED; + } + + m_gameFolderArchives.push_back(path); + return generals ? ARCHIVE_PRIORITY_GAME_FOLDER_GENERALS : ARCHIVE_PRIORITY_GAME_FOLDER; +} + +Bool ArchiveFileSystem::parseMountEntry(const AsciiString& text, ArchiveMountEntry* entry, AsciiString* reason) +{ + auto fail = [reason](const char* why) + { + if (reason != nullptr) + *reason = why; + return FALSE; + }; + + std::string value = text.str(); + // trim + while (!value.empty() && isspace(static_cast(value.back()))) + value.pop_back(); + size_t start = 0; + while (start < value.size() && isspace(static_cast(value[start]))) + ++start; + value = value.substr(start); + if (value.empty()) + return fail("empty"); + + for (char& c : value) + { + if (c == '/') + c = '\\'; + if (static_cast(c) < 32 || c == '<' || c == '>' || c == '"' || c == '|') + return fail("invalid character"); + } -#if defined(DEBUG_LOGGING) && ENABLE_FILESYSTEM_LOGGING + ArchiveMountEntry::Base base; + std::string path; + std::string lowerPrefix = value.substr(0, 5); + for (char& c : lowerPrefix) + c = static_cast(tolower(static_cast(c))); + if (lowerPrefix == "game:") + { + base = ArchiveMountEntry::BASE_GAME; + path = value.substr(5); + } + else if (lowerPrefix == "user:") + { + base = ArchiveMountEntry::BASE_USER; + path = value.substr(5); + } + else if (value.size() > 3 && isalpha(static_cast(value[0])) && value[1] == ':' && value[2] == '\\') + { + base = ArchiveMountEntry::BASE_ABSOLUTE; + path = value; + } + else if (value.size() > 2 && value[0] == '\\' && value[1] == '\\') + { + base = ArchiveMountEntry::BASE_ABSOLUTE; + path = value; + } + else + { + return fail("needs the prefix game: or user:, or an absolute path"); + } + + if (base != ArchiveMountEntry::BASE_ABSOLUTE) + { + if (path.empty() || path[0] == '\\' || path.find(':') != std::string::npos) + return fail("invalid path"); + } + + // no parent folders + size_t pos = 0; + while (pos <= path.size()) + { + size_t next = path.find('\\', pos); + if (next == std::string::npos) + next = path.size(); + if (path.substr(pos, next - pos) == "..") + return fail("parent folder"); + pos = next + 1; + } + + const size_t lastSeparator = path.find_last_of('\\'); + const std::string lastComponent = lastSeparator == std::string::npos ? path : path.substr(lastSeparator + 1); + const std::string directory = lastSeparator == std::string::npos ? std::string() : path.substr(0, lastSeparator + 1); + if (directory.find_first_of("*?") != std::string::npos) + return fail("wildcards are only allowed in the file name"); + + std::string lowerLast = lastComponent; + for (char& c : lowerLast) + c = static_cast(tolower(static_cast(c))); + + ArchiveMountEntry::Kind kind; + if (lastComponent.empty()) + { + // a folder: every BIG in it + if (directory.empty() || directory == "\\\\") + return fail("invalid path"); + kind = ArchiveMountEntry::KIND_BIG_GLOB; + path += "*.big"; + } + else if (lowerLast.size() < 5 || lowerLast.compare(lowerLast.size() - 4, 4, ".big") != 0) + { + return fail("not a BIG file, a BIG folder or a *.big pattern"); + } + else if (lastComponent.find_first_of("*?") != std::string::npos) + { + kind = ArchiveMountEntry::KIND_BIG_GLOB; + } + else + { + kind = ArchiveMountEntry::KIND_BIG; + } + + if (entry != nullptr) + { + entry->m_kind = kind; + entry->m_base = base; + entry->m_path = path.c_str(); + } + return TRUE; +} + +void ArchiveFileSystem::mountListedArchive(const ArchiveMountEntry& entry, Int priority, const PathExcludeList* excludes) +{ + std::string fullPath = entry.m_path.str(); + if (entry.m_base == ArchiveMountEntry::BASE_USER) + fullPath = std::string(TheGlobalData->getPath_UserData().str()) + fullPath; + + // the files to mount, in name order + std::vector bigPaths; + if (entry.m_kind == ArchiveMountEntry::KIND_BIG) + { + bigPaths.push_back(fullPath); + } + else + { + const size_t lastSeparator = fullPath.find_last_of('\\'); + const std::string directory = lastSeparator == std::string::npos ? std::string() : fullPath.substr(0, lastSeparator + 1); + const std::string mask = lastSeparator == std::string::npos ? fullPath : fullPath.substr(lastSeparator + 1); + FilenameList files; + TheLocalFileSystem->getFileListInDirectory(AsciiString(directory.c_str()), "", AsciiString(mask.c_str()), files, FALSE); + for (FilenameListIter it = files.begin(); it != files.end(); ++it) + bigPaths.push_back(it->str()); + } + + Int mounted = 0; + for (const std::string& bigPath : bigPaths) + { + if (!TheLocalFileSystem->doesFileExist(bigPath.c_str())) + continue; + if (excludes != nullptr && !excludes->isEmpty() && excludes->matches(PathExcludeList::makeKey(bigPath.c_str(), TheGlobalData->getPath_UserData().str()))) { - const stl::const_range range = stl::get_range(dirInfo->m_files, token, 0); - if (range.distance() >= 2) - { - ArchivedFileLocationMap::const_iterator rangeIt0; - ArchivedFileLocationMap::const_iterator rangeIt1; - - if (overwrite) - { - rangeIt0 = range.begin; - rangeIt1 = std::next(rangeIt0); - - DEBUG_LOG(("ArchiveFileSystem::loadIntoDirectoryTree - adding file %s, archived in %s, overwriting same file in %s", - it->str(), - rangeIt0->second->getName().str(), - rangeIt1->second->getName().str() - )); - } - else - { - rangeIt1 = std::prev(range.end); - rangeIt0 = std::prev(rangeIt1); - - DEBUG_LOG(("ArchiveFileSystem::loadIntoDirectoryTree - adding file %s, archived in %s, overwritten by same file in %s", - it->str(), - rangeIt1->second->getName().str(), - rangeIt0->second->getName().str() - )); - } - } - else - { - DEBUG_LOG(("ArchiveFileSystem::loadIntoDirectoryTree - adding file %s, archived in %s", it->str(), archiveFile->getName().str())); - } + NetworkLog(ELogVerbosity::LOG_RELEASE, "Archive %s is excluded", bigPath.c_str()); + continue; + } + if (m_archiveFileMap.find(AsciiString(bigPath.c_str())) != m_archiveFileMap.end()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Archive %s is already mounted", bigPath.c_str()); + continue; } -#endif + ArchiveFile* archiveFile = openArchiveFile(bigPath.c_str()); + if (archiveFile == nullptr) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Archive %s could not be opened", bigPath.c_str()); + continue; + } + loadIntoDirectoryTree(archiveFile, priority); + m_archiveFileMap[AsciiString(bigPath.c_str())] = archiveFile; + ++mounted; + } + NetworkLog(ELogVerbosity::LOG_RELEASE, "Archive %s at priority %d: %d archives", fullPath.c_str(), priority, mounted); +} - it++; +void ArchiveFileSystem::mountDataPackLayer(const char* layer, Bool enabled, const std::vector& includePaths, const PathExcludeList* excludes, Int priority) +{ + if (!enabled) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Data pack layer %s is disabled", layer); + return; + } + + NetworkLog(ELogVerbosity::LOG_RELEASE, "Data pack layer %s: %d include_paths", layer, static_cast(includePaths.size())); + for (size_t i = 0; i < includePaths.size(); ++i) + { + ArchiveMountEntry entry; + AsciiString reason; + if (!parseMountEntry(AsciiString(includePaths[i].c_str()), &entry, &reason)) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Data pack layer %s: path ignored (%s): %s", layer, reason.str(), includePaths[i].c_str()); + continue; + } + mountListedArchive(entry, priority + static_cast(i), excludes); } } diff --git a/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp b/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp index d5fb1ec8fd6..7b84df515a0 100644 --- a/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp +++ b/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp @@ -92,12 +92,20 @@ void BIGFileSystem::init() { #if RTS_ZEROHOUR // load original Generals assets + // The Steam install ships them in ZH_Generals, which the scan above already mounted. Mounting the Generals install as well would only add its mods. + if (TheLocalFileSystem->doesFileExist("ZH_Generals\\INI.big")) { + m_generalsBaseSource = "ZH_Generals folder"; + return; + } + AsciiString installPath; GetStringFromGeneralsRegistry("", "InstallPath", installPath ); //@todo this will need to be ramped up to a crash for release DEBUG_ASSERTCRASH(!installPath.isEmpty(), ("Be 1337! Go install Generals!")); - if (!installPath.isEmpty()) + if (!installPath.isEmpty()) { + m_generalsBaseSource.format("Generals install folder %s", installPath.str()); loadArchiveFilesFromDirectory(installPath, "*.big"); + } #endif } @@ -252,11 +260,23 @@ Bool BIGFileSystem::loadArchiveFilesFromDirectory(AsciiString dir, AsciiString f } #endif +#if RTS_ZEROHOUR + const Bool zeroHour = TRUE; +#else + const Bool zeroHour = FALSE; +#endif + // The first scan is the game folder, the second one the Generals install: the retail archives are the bottom of the lookup + const Int priority = overwrite ? getNextModPriority() : getGameFolderArchivePriority((*it), dir.isNotEmpty(), zeroHour); + if (priority == ARCHIVE_PRIORITY_EXCLUDED) { + it++; + continue; + } + ArchiveFile *archiveFile = openArchiveFile((*it).str()); if (archiveFile != nullptr) { DEBUG_LOG(("BIGFileSystem::loadArchiveFilesFromDirectory - loading %s into the directory tree.", (*it).str())); - loadIntoDirectoryTree(archiveFile, overwrite); + loadIntoDirectoryTree(archiveFile, priority); m_archiveFileMap[(*it)] = archiveFile; DEBUG_LOG(("BIGFileSystem::loadArchiveFilesFromDirectory - %s inserted into the archive file map.", (*it).str())); actuallyAdded = TRUE; @@ -270,26 +290,21 @@ Bool BIGFileSystem::loadArchiveFilesFromDirectory(AsciiString dir, AsciiString f void BIGFileSystem::loadMods() { -#if defined(GENERALS_ONLINE) && defined(GENERALS_ONLINE_COMMUNITY_PATCH_CHANGES) - // load community data patch BIG - if (NGMP_OnlineServicesManager::Settings.DataPacks_UseCommunityPatch() - && !TheGlobalData->m_commandLineData.isCommunityDataPatchDisabled()) - { - std::string strBigPath = std::format("{}GeneralsOnlineGameData/500_900_CommunityPatch_CoreINI.big", TheGlobalData->getPath_UserData().str()); - bool bLoaded = false; - if (TheLocalFileSystem->doesFileExist(strBigPath.c_str())) - { - ArchiveFile* archiveFile = openArchiveFile(strBigPath.c_str()); - if (archiveFile != nullptr) - { - // Sorted by filename so the patch respects the addon number order of BIGs in the game folder. - loadIntoDirectoryTree(archiveFile, FALSE, TRUE); - m_archiveFileMap[AsciiString(strBigPath.c_str())] = archiveFile; - bLoaded = true; - } - } - NetworkLog(ELogVerbosity::LOG_RELEASE, "Loaded community patch (%s): %d", strBigPath.c_str(), bLoaded); - } +#if defined(GENERALS_ONLINE) + // Order, highest first: loose files of the game folder, -mod and -modDir, the BIGs of the game folders that are not retail (user.exclude_paths removes some), + // the data pack layers of settings.json (user, generals_online, community_patch), the retail archives. The BIGs of the game folders were mounted by init(). + NetworkLog(ELogVerbosity::LOG_RELEASE, "Generals data: %s", m_generalsBaseSource.isEmpty() ? "none" : m_generalsBaseSource.str()); + NetworkLog(ELogVerbosity::LOG_RELEASE, "Game folder archives above the data pack layers: %d", static_cast(m_gameFolderArchives.size())); + for (const AsciiString& gameFolderArchive : m_gameFolderArchives) + NetworkLog(ELogVerbosity::LOG_RELEASE, " %s", gameFolderArchive.str()); + + const GenOnlineSettings& settings = NGMP_OnlineServicesManager::Settings; + mountDataPackLayer("user", settings.DataPacks_User().m_enabled, settings.DataPacks_User().m_includePaths, &TheFileSystem->getExcludedPaths(), ARCHIVE_PRIORITY_USER); + mountDataPackLayer("generals_online", settings.DataPacks_GeneralsOnline().m_enabled, settings.DataPacks_GeneralsOnline().m_includePaths, nullptr, ARCHIVE_PRIORITY_GENERALS_ONLINE); + + // -disableCommunityDataPatch skips the community_patch layer for this launch only, the setting is not changed + const Bool communityPatchEnabled = settings.DataPacks_CommunityPatch().m_enabled && !TheGlobalData->m_commandLineData.isCommunityDataPatchDisabled(); + mountDataPackLayer("community_patch", communityPatchEnabled, settings.DataPacks_CommunityPatch().m_includePaths, nullptr, ARCHIVE_PRIORITY_COMMUNITY_PATCH); #endif if (TheGlobalData->m_modBIG.isNotEmpty()) @@ -298,7 +313,7 @@ void BIGFileSystem::loadMods() if (archiveFile != nullptr) { DEBUG_LOG(("BIGFileSystem::loadMods - loading %s into the directory tree.", TheGlobalData->m_modBIG.str())); - loadIntoDirectoryTree(archiveFile, TRUE); + loadIntoDirectoryTree(archiveFile, getNextModPriority()); m_archiveFileMap[TheGlobalData->m_modBIG] = archiveFile; DEBUG_LOG(("BIGFileSystem::loadMods - %s inserted into the archive file map.", TheGlobalData->m_modBIG.str())); } diff --git a/Core/GameEngine/Source/Common/System/FileSystem.cpp b/Core/GameEngine/Source/Common/System/FileSystem.cpp index b8e4c4695b6..e4df60933a4 100644 --- a/Core/GameEngine/Source/Common/System/FileSystem.cpp +++ b/Core/GameEngine/Source/Common/System/FileSystem.cpp @@ -56,6 +56,8 @@ #include "Lib/PathUtil.h" +#include + DECLARE_PERF_TIMER(FileSystem) @@ -118,6 +120,184 @@ FileSystem *TheFileSystem = nullptr; //---------------------------------------------------------------------------- +//============================================================================ +// PathExcludeList +//============================================================================ + +static std::string normalizeExcludeText(const std::string& text) +{ + std::string value = text; + for (size_t i = 0; i < value.size(); ++i) + { + if (value[i] == '/') + value[i] = '\\'; + value[i] = static_cast(tolower(static_cast(value[i]))); + } + return value; +} + +static Bool isAbsoluteExcludePath(const std::string& value) +{ + return (value.size() > 2 && isalpha(static_cast(value[0])) && value[1] == ':' && value[2] == '\\') + || (value.size() > 2 && value[0] == '\\' && value[1] == '\\'); +} + +static std::string stripCurrentFolder(const std::string& value) +{ + size_t start = 0; + while (start + 2 <= value.size() && value.compare(start, 2, ".\\") == 0) + start += 2; + return value.substr(start); +} + +Bool PathExcludeList::parse(const std::string& text, std::string* pattern, AsciiString* reason) +{ + std::string value = text; + while (!value.empty() && isspace(static_cast(value[value.size() - 1]))) + value.erase(value.size() - 1); + size_t start = 0; + while (start < value.size() && isspace(static_cast(value[start]))) + ++start; + value = normalizeExcludeText(value.substr(start)); + + if (value.empty()) + { + if (reason != nullptr) + *reason = "empty"; + return FALSE; + } + for (size_t i = 0; i < value.size(); ++i) + { + const unsigned char c = static_cast(value[i]); + if (c < 32 || c == '<' || c == '>' || c == '"' || c == '|') + { + if (reason != nullptr) + *reason = "invalid character"; + return FALSE; + } + } + + std::string rest; + std::string prefix; + if (value.compare(0, 5, "game:") == 0 || value.compare(0, 5, "user:") == 0) + { + prefix = value.substr(0, 5); + rest = stripCurrentFolder(value.substr(5)); + if (rest.empty() || rest[0] == '\\' || rest.find(':') != std::string::npos) + { + if (reason != nullptr) + *reason = "invalid path"; + return FALSE; + } + } + else if (isAbsoluteExcludePath(value)) + { + rest = value; + } + else + { + if (reason != nullptr) + *reason = "needs the prefix game: or user:, or an absolute path"; + return FALSE; + } + + // no parent folders + size_t pos = 0; + while (pos <= rest.size()) + { + size_t next = rest.find('\\', pos); + if (next == std::string::npos) + next = rest.size(); + if (next - pos == 2 && rest.compare(pos, 2, "..") == 0) + { + if (reason != nullptr) + *reason = "parent folder"; + return FALSE; + } + pos = next + 1; + } + + // a folder: everything in it + if (rest[rest.size() - 1] == '\\') + rest += "*"; + + if (pattern != nullptr) + *pattern = prefix + rest; + return TRUE; +} + +std::string PathExcludeList::makeKey(const Char* path) +{ + std::string value = stripCurrentFolder(normalizeExcludeText(path)); + if (isAbsoluteExcludePath(value)) + return value; + return "game:" + value; +} + +std::string PathExcludeList::makeKey(const Char* path, const Char* userDataPath) +{ + std::string value = normalizeExcludeText(path); + std::string userData = normalizeExcludeText(userDataPath); + if (!userData.empty() && userData[userData.size() - 1] != '\\') + userData += "\\"; + if (!userData.empty() && value.size() > userData.size() && value.compare(0, userData.size(), userData) == 0) + return "user:" + value.substr(userData.size()); + return makeKey(path); +} + +void PathExcludeList::set(const std::vector& entries) +{ + m_patterns.clear(); + for (size_t i = 0; i < entries.size(); ++i) + { + std::string pattern; + if (parse(entries[i], &pattern, nullptr)) + m_patterns.push_back(pattern); + } +} + +// * matches any text, ? one character +static Bool matchesExcludePattern(const char* pattern, const char* text) +{ + const char* starPattern = nullptr; + const char* starText = nullptr; + while (*text != '\0') + { + if (*pattern == '*') + { + starPattern = ++pattern; + starText = text; + } + else if (*pattern == '?' || *pattern == *text) + { + ++pattern; + ++text; + } + else if (starPattern != nullptr) + { + pattern = starPattern; + text = ++starText; + } + else + { + return FALSE; + } + } + while (*pattern == '*') + ++pattern; + return *pattern == '\0'; +} + +Bool PathExcludeList::matches(const std::string& key) const +{ + for (size_t i = 0; i < m_patterns.size(); ++i) + { + if (matchesExcludePattern(m_patterns[i].c_str(), key.c_str())) + return TRUE; + } + return FALSE; +} + //============================================================================ // FileSystem::FileSystem //============================================================================ @@ -179,14 +359,17 @@ File* FileSystem::openFile( const Char *filename, Int access, size_t bufferSize if ( TheLocalFileSystem != nullptr ) { + // an excluded loose file does not exist, unless the file is written + const Bool excluded = (access & (File::WRITE | File::APPEND | File::CREATE | File::TRUNCATE)) == 0 && isLooseFileExcluded(filename); + if (instance != 0) { - if (TheLocalFileSystem->doesFileExist(filename)) + if (!excluded && TheLocalFileSystem->doesFileExist(filename)) { --instance; } } - else + else if (!excluded) { file = TheLocalFileSystem->openFile( filename, access, bufferSize ); @@ -242,7 +425,7 @@ Bool FileSystem::doesFileExist(const Char *filename, FileInstance instance) cons } #endif - if (TheLocalFileSystem->doesFileExist(filename)) + if (!isLooseFileExcluded(filename) && TheLocalFileSystem->doesFileExist(filename)) { if (instance == 0) { @@ -286,7 +469,20 @@ Bool FileSystem::doesFileExist(const Char *filename, FileInstance instance) cons void FileSystem::getFileListInDirectory(const AsciiString& directory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const { USE_PERF_TIMER(FileSystem) - TheLocalFileSystem->getFileListInDirectory(AsciiString::TheEmptyString, directory, searchName, filenameList, searchSubdirectories); + if (m_excludedPaths.isEmpty()) + { + TheLocalFileSystem->getFileListInDirectory(AsciiString::TheEmptyString, directory, searchName, filenameList, searchSubdirectories); + } + else + { + FilenameList localFiles; + TheLocalFileSystem->getFileListInDirectory(AsciiString::TheEmptyString, directory, searchName, localFiles, searchSubdirectories); + for (FilenameListIter it = localFiles.begin(); it != localFiles.end(); ++it) + { + if (!isLooseFileExcluded(it->str())) + filenameList.insert(*it); + } + } TheArchiveFileSystem->getFileListInDirectory(AsciiString::TheEmptyString, directory, searchName, filenameList, searchSubdirectories); } @@ -304,7 +500,7 @@ Bool FileSystem::getFileInfo(const AsciiString& filename, FileInfo *fileInfo, Fi } memset(fileInfo, 0, sizeof(*fileInfo)); - if (TheLocalFileSystem->getFileInfo(filename, fileInfo)) { + if (!isLooseFileExcluded(filename.str()) && TheLocalFileSystem->getFileInfo(filename, fileInfo)) { if (instance == 0) { return TRUE; } diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.h index 0bcc1a2c3e3..8ac6d4fb329 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.h @@ -10,6 +10,13 @@ enum EHTTPVersion HTTP_VERSION_3_0 }; +// One layer of data_packs in settings.json: BIG paths in priority order, the first one wins +struct GenOnlineDataPackLayer +{ + bool m_enabled = true; + std::vector m_includePaths; +}; + class GenOnlineSettings { public: @@ -70,7 +77,10 @@ class GenOnlineSettings Load(); } - bool DataPacks_UseCommunityPatch() const { return m_DataPacks_UseCommunityPatch; } + const GenOnlineDataPackLayer& DataPacks_User() const { return m_DataPacks_User; } + const GenOnlineDataPackLayer& DataPacks_GeneralsOnline() const { return m_DataPacks_GeneralsOnline; } + const GenOnlineDataPackLayer& DataPacks_CommunityPatch() const { return m_DataPacks_CommunityPatch; } + const std::vector& DataPacks_UserExcludePaths() const { return m_DataPacks_UserExcludePaths; } bool Network_UseAlternativeEndpoint() const { return m_Network_UseAlternativeEndpoint; } EHTTPVersion Network_GetHTTPVersion() const { return m_Network_HTTPVersion; } @@ -139,7 +149,11 @@ class GenOnlineSettings bool m_Social_Notification_PlayerSendsRequest_Menus = true; bool m_Social_Notification_PlayerSendsRequest_Gameplay = true; - bool m_DataPacks_UseCommunityPatch = true; + // BIG archives above the retail ones. The layers load in this order, highest priority first. A missing or invalid field keeps its default. + GenOnlineDataPackLayer m_DataPacks_User; + std::vector m_DataPacks_UserExcludePaths; // loose files of the game folder, BIGs of the game folders and BIGs of the user layer that are left out + GenOnlineDataPackLayer m_DataPacks_GeneralsOnline = { true, { "user:GeneralsOnlineGameData/GeneralsOnline_Core.big" } }; + GenOnlineDataPackLayer m_DataPacks_CommunityPatch = { true, { "user:GeneralsOnlineGameData/500_900_CommunityPatch_CoreINI.big" } }; std::string m_Plugins_Anticheat = std::string(); diff --git a/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp b/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp index 0c497745de0..63e64f71f76 100644 --- a/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp +++ b/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp @@ -504,6 +504,10 @@ void GameEngine::init() #endif///////////////////////////////////////////////////////////////////////////////////////////// + // NGMP_CHANGE: Init our settings before the archives are mounted, the data_packs layers decide which ones. Needs TheGlobalData for the user data path. + NGMP_OnlineServicesManager::Settings.Initialize(); + TheFileSystem->setExcludedPaths(NGMP_OnlineServicesManager::Settings.DataPacks_UserExcludePaths()); + initSubsystem(TheArchiveFileSystem, "TheArchiveFileSystem", createArchiveFileSystem(), nullptr); // this MUST come after TheLocalFileSystem creation #ifdef DUMP_PERF_STATS/////////////////////////////////////////////////////////////////////////// @@ -538,9 +542,6 @@ void GameEngine::init() // special-case: parse command-line parameters after loading global data CommandLine::parseCommandLineForEngineInit(); - // NGMP_CHANGE: Init our settings before loadMods, which reads DataPacks_UseCommunityPatch. Needs TheGlobalData for the user data path. - NGMP_OnlineServicesManager::Settings.Initialize(); - TheArchiveFileSystem->loadMods(); // doesn't require resets so just create a single instance here. diff --git a/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GameWindowManagerScript.cpp b/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GameWindowManagerScript.cpp index d588e90cc31..d1d2da5bea0 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GameWindowManagerScript.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GameWindowManagerScript.cpp @@ -2726,26 +2726,6 @@ GameWindow *GameWindowManager::winCreateFromScript( AsciiString filenameString, else strlcpy(filepath, filename, ARRAY_SIZE(filepath)); -#if defined(GENERALS_ONLINE) - // check Generals Online first - char gofilepath[_MAX_PATH] = "GeneralsOnlineGameData\\"; - if (strchr(filename, '\\') == NULL) - sprintf(gofilepath, "GeneralsOnlineGameData\\%s", filename); - else - strcpy(gofilepath, filename); - - inFile = TheFileSystem->openFile(gofilepath, File::READ); - if (inFile == NULL) - { - // fall back to game archive - inFile = TheFileSystem->openFile(filepath, File::READ); - if (inFile == NULL) - { - DEBUG_LOG(("WinCreateFromScript: Cannot access file '%s'.\n", filename)); - return NULL; - } - } -#else // Open the input file inFile = TheFileSystem->openFile(filepath, File::READ); if (inFile == nullptr) @@ -2753,7 +2733,6 @@ GameWindow *GameWindowManager::winCreateFromScript( AsciiString filenameString, DEBUG_LOG(( "WinCreateFromScript: Cannot access file '%s'.", filename )); return nullptr; } -#endif // read into memory inFile=inFile->convertToRAMFile(); diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.cpp index 347c65de981..b8e3ca9bfbc 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.cpp @@ -1,4 +1,5 @@ #include "GameNetwork/GeneralsOnline/GeneralsOnline_Settings.h" +#include "Common/ArchiveFileSystem.h" #include "../json.hpp" #include "../OnlineServices_LobbyInterface.h" #include "../OnlineServices_Init.h" @@ -37,7 +38,13 @@ #define SETTINGS_KEY_PLUGINS_ANTICHEAT "anticheat" #define SETTINGS_KEY_DATAPACKS "data_packs" -#define SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH "use_community_data_patch" +#define SETTINGS_KEY_DATAPACKS_USER "user" +#define SETTINGS_KEY_DATAPACKS_GENERALS_ONLINE "generals_online" +#define SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH "community_patch" +#define SETTINGS_KEY_DATAPACKS_ENABLED "enabled" +#define SETTINGS_KEY_DATAPACKS_INCLUDE_PATHS "include_paths" +#define SETTINGS_KEY_DATAPACKS_EXCLUDE_PATHS "exclude_paths" +#define SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH_LEGACY "use_community_data_patch" // before the layers, read once and dropped on save #define SETTINGS_FILENAME_LEGACY "GeneralsOnline_settings.json" #define SETTINGS_FILENAME "settings.json" @@ -94,6 +101,89 @@ std::string GenOnlineSettings::GetAnticheatPlugin() const return m_Plugins_Anticheat; } +// Reads a list of paths of data_packs, an entry that is not valid is dropped. Returns FALSE and keeps the list when the field is missing or not a list. +static bool LoadDataPackPaths(const nlohmann::json& jsonLayer, const char* layerName, const char* key, bool excludes, std::vector& paths) +{ + if (!jsonLayer.contains(key)) + { + return false; + } + + if (!jsonLayer[key].is_array()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Data pack layer %s: %s is not a list, using the default", layerName, key); + return false; + } + + paths.clear(); + for (const auto& path : jsonLayer[key]) + { + AsciiString reason; + bool bValid = false; + if (path.is_string()) + { + bValid = excludes ? PathExcludeList::parse(path.get(), nullptr, &reason) : ArchiveFileSystem::parseMountEntry(AsciiString(path.get().c_str()), nullptr, &reason); + } + + if (bValid) + { + paths.push_back(path.get()); + } + else + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Data pack layer %s: %s entry dropped from the settings (%s)", layerName, key, path.is_string() ? reason.str() : "not a string"); + } + } + return true; +} + +// Reads one layer of data_packs. A missing or invalid field keeps what the layer holds, its default. excludePaths is only read for the user layer. +static void LoadDataPackLayer(const nlohmann::json& dataPacks, const char* name, GenOnlineDataPackLayer& layer, std::vector* excludePaths) +{ + if (!dataPacks.contains(name)) + { + return; + } + + const nlohmann::json& jsonLayer = dataPacks[name]; + if (!jsonLayer.is_object()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Data pack layer %s in the settings is not an object, using the default", name); + return; + } + + if (jsonLayer.contains(SETTINGS_KEY_DATAPACKS_ENABLED)) + { + if (jsonLayer[SETTINGS_KEY_DATAPACKS_ENABLED].is_boolean()) + { + layer.m_enabled = jsonLayer[SETTINGS_KEY_DATAPACKS_ENABLED].get(); + } + else + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Data pack layer %s: enabled is not a boolean, using the default", name); + } + } + + LoadDataPackPaths(jsonLayer, name, SETTINGS_KEY_DATAPACKS_INCLUDE_PATHS, false, layer.m_includePaths); + + if (excludePaths != nullptr) + { + LoadDataPackPaths(jsonLayer, name, SETTINGS_KEY_DATAPACKS_EXCLUDE_PATHS, true, *excludePaths); + } +} + +static nlohmann::json DataPackLayerToJson(const GenOnlineDataPackLayer& layer, const std::vector* excludePaths) +{ + nlohmann::json jsonLayer = nlohmann::json::object(); + jsonLayer[SETTINGS_KEY_DATAPACKS_ENABLED] = layer.m_enabled; + jsonLayer[SETTINGS_KEY_DATAPACKS_INCLUDE_PATHS] = layer.m_includePaths; + if (excludePaths != nullptr) + { + jsonLayer[SETTINGS_KEY_DATAPACKS_EXCLUDE_PATHS] = *excludePaths; + } + return jsonLayer; +} + void GenOnlineSettings::Load(void) { char GameDir[MAX_PATH + 1] = {}; @@ -317,11 +407,18 @@ void GenOnlineSettings::Load(void) if (jsonSettings.contains(SETTINGS_KEY_DATAPACKS)) { - auto datapackSettings = jsonSettings[SETTINGS_KEY_DATAPACKS]; + const nlohmann::json& datapackSettings = jsonSettings[SETTINGS_KEY_DATAPACKS]; + + LoadDataPackLayer(datapackSettings, SETTINGS_KEY_DATAPACKS_USER, m_DataPacks_User, &m_DataPacks_UserExcludePaths); + LoadDataPackLayer(datapackSettings, SETTINGS_KEY_DATAPACKS_GENERALS_ONLINE, m_DataPacks_GeneralsOnline, nullptr); + LoadDataPackLayer(datapackSettings, SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH, m_DataPacks_CommunityPatch, nullptr); - if (datapackSettings.contains(SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH)) + // a file from before the layers: the old community patch switch becomes community_patch.enabled + if (!datapackSettings.contains(SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH) + && datapackSettings.contains(SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH_LEGACY) + && datapackSettings[SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH_LEGACY].is_boolean()) { - m_DataPacks_UseCommunityPatch = datapackSettings[SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH]; + m_DataPacks_CommunityPatch.m_enabled = datapackSettings[SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH_LEGACY].get(); } } } @@ -443,7 +540,9 @@ void GenOnlineSettings::Save() { SETTINGS_KEY_DATAPACKS, { - {SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH, m_DataPacks_UseCommunityPatch} + {SETTINGS_KEY_DATAPACKS_USER, DataPackLayerToJson(m_DataPacks_User, &m_DataPacks_UserExcludePaths)}, + {SETTINGS_KEY_DATAPACKS_GENERALS_ONLINE, DataPackLayerToJson(m_DataPacks_GeneralsOnline, nullptr)}, + {SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH, DataPackLayerToJson(m_DataPacks_CommunityPatch, nullptr)} } } }; diff --git a/scripts/pack_go_data.py b/scripts/pack_go_data.py new file mode 100644 index 00000000000..a084b052273 --- /dev/null +++ b/scripts/pack_go_data.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +r"""Pack GeneralsOnline/GameData into the Generals Online fix pack BIG. + +Usage: pack_go_data.py [output.big] + +The folder layout inside GameData is the layout inside the BIG (Window\..., Data\INI\..., Art\Textures\...). +The updater ships the result as \GeneralsOnlineGameData\GeneralsOnline_Core.big. + +The client mounts it through the generals_online layer of data_packs in settings.json. From the top, the lookup is: loose files in the +game folder, -mod and -modDir, the BIGs in the game folders that are not retail, then the layers user, generals_online and community_patch, +then the retail archives. Every layer has enabled and include_paths, which are mounted in list order (the first one wins). An entry is +game:, user: or an absolute path: a BIG, a folder of BIGs or a *.big pattern. + + "data_packs": { + "user": { "enabled": true, "include_paths": [], "exclude_paths": [] }, + "generals_online": { "enabled": true, "include_paths": [ "user:GeneralsOnlineGameData/GeneralsOnline_Core.big" ] }, + "community_patch": { "enabled": true, "include_paths": [ "user:GeneralsOnlineGameData/500_900_CommunityPatch_CoreINI.big" ] } + } + +user.exclude_paths leaves things out: loose files in the game folder (game:Data/INI/*), the BIGs in the game folders that are not retail +(game:340_ControlBarPro*.big, game:ZH_Generals/*.big) and the include_paths of the user layer itself. * matches any text including folders, +a trailing slash is a folder. Retail archives are never excluded. user.enabled false only turns off user.include_paths. + +The old data_packs.use_community_data_patch switch is read as community_patch.enabled when community_patch is missing. +-disableCommunityDataPatch skips the community_patch layer for one launch. +""" +import os +import struct +import sys + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "GeneralsOnline", "GameData") +OUT = sys.argv[1] if len(sys.argv) > 1 else "GeneralsOnline_Core.big" + + +def main(): + files = [] + for dirpath, _, names in os.walk(ROOT): + for name in names: + full = os.path.join(dirpath, name) + rel = os.path.relpath(full, ROOT).replace("/", "\\") + files.append((rel, full)) + files.sort(key=lambda f: f[0].lower()) + if not files: + sys.exit("nothing to pack in " + ROOT) + + names = [rel.encode("ascii") + b"\0" for rel, _ in files] + header = 16 + sum(8 + len(n) for n in names) + table = b"" + body = b"" + offset = header + for (rel, full), n in zip(files, names): + with open(full, "rb") as f: + data = f.read() + table += struct.pack(">II", offset, len(data)) + n + body += data + offset += len(data) + # BIGF: magic, total size (little endian), file count and data start (big endian) + with open(OUT, "wb") as f: + f.write(b"BIGF" + struct.pack("II", len(files), header) + table + body) + print("packed %d files into %s" % (len(files), OUT)) + + +main() From 8d76c06dd00cdd2294fd369d04772ccf857a6687 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Sun, 18 Oct 2026 05:37:02 +0200 Subject: [PATCH 15/17] fix(online): Preserve WebSocket sends under backpressure and read every queued frame --- .../GeneralsOnline/OnlineServices_Init.h | 16 +- .../OnlineServices_RoomsInterface.cpp | 236 ++++++++++++------ 2 files changed, 176 insertions(+), 76 deletions(-) diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index 2096e7b3623..fbe31cc50f2 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -226,6 +226,8 @@ class WebSocket } private: + CURLcode ReceiveOneFrame(); + CURL* m_pCurlWS = nullptr; CURLM* m_pMulti = nullptr; struct curl_slist* m_pHeaders = nullptr; @@ -261,7 +263,19 @@ class WebSocket // Outbound messages queued for Tick() to flush. std::mutex m_outboundQueueMutex; - std::vector m_vecQueuedOutboungMsgs; + struct OutboundMessage + { + std::string payload; + bool heartbeat; + }; + std::vector m_vecQueuedOutboungMsgs; + size_t m_queuedOutboundBytes = 0; + size_t m_outboundOffset = 0; + bool m_heartbeatQueued = false; + bool m_acceptOutbound = true; + void QueueOutbound(std::string payload, bool heartbeat); + void ClearOutbound(); + CURLcode FlushOutbound(); std::mutex m_pendingSignalsMutex; std::queue> m_pendingSignals; diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp index 07e13867223..bf388eecd86 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp @@ -9,10 +9,10 @@ #include "../HTTP/HTTPManager.h" #include "GameNetwork/GameSpy/PeerDefs.h" -// one clock for every websocket timestamp; utc_clock also loads the time zone database on MSVC +// steady clock, so deadlines do not move with system clock changes static int64_t NowMs() { - return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); } // ----------------------------- @@ -138,20 +138,18 @@ WebSocket::~WebSocket() int WebSocket::Ping() { - size_t sent; - CURLcode result = curl_ws_send(m_pCurlWS, "wsping", strlen("wsping"), &sent, 0, - CURLWS_PING); + if (!m_bConnected) + { + return CURLE_AGAIN; + } + // keep the app heartbeat for older services, no raw PING, at most one queued under backpressure nlohmann::json j; j["msg_id"] = EWebSocketMessageID::PING; - std::string strBody = j.dump(); - - Send(strBody.c_str()); - - return (int)result; + QueueOutbound(j.dump(), true); + return CURLE_OK; } - void WebSocket::Connect(const char* url, bool bIsReconnect, std::function fnWebsocketConnectedCallback) { if (m_bConnected) @@ -275,34 +273,23 @@ void WebSocket::SendData_JoinNetworkRoom(int roomID, uint64_t requestID) void WebSocket::Disconnect() { + EndReconnect(); if (!m_bConnected) { + ClearOutbound(); return; } if (m_pCurlWS != nullptr) { - // best-effort flush of anything queued since the last Tick() before closing - std::vector outboundBatch; + // best effort, never insert CLOSE into a partially sent frame + const CURLcode result = FlushOutbound(); + if (result == CURLE_OK) { - std::scoped_lock lock(m_outboundQueueMutex); - outboundBatch.swap(m_vecQueuedOutboungMsgs); + size_t sent = 0; + (void)curl_ws_send(m_pCurlWS, "", 0, &sent, 0, CURLWS_CLOSE); } - for (std::string& strPayload : outboundBatch) - { - size_t sentPayload; - CURLcode sendResult = curl_ws_send(m_pCurlWS, strPayload.c_str(), strPayload.length(), &sentPayload, 0, CURLWS_BINARY); - if (sendResult != CURLE_OK) - { - NetworkLog(ELogVerbosity::LOG_RELEASE, "[WebSocket] Disconnect: failed to flush queued message: %s", curl_easy_strerror(sendResult)); - } - } - - // send close - size_t sent; - (void)curl_ws_send(m_pCurlWS, "", 0, &sent, 0, CURLWS_CLOSE); - // release headers if (m_pHeaders != nullptr) { @@ -323,23 +310,104 @@ void WebSocket::Disconnect() m_pCurlWS = nullptr; } + ClearOutbound(); m_vecWSPartialBuffer.clear(); m_bConnected = false; } void WebSocket::Send(const char* send_payload) { - // Thread-safe; always queues. Tick() flushes on the main thread. + QueueOutbound(std::string(send_payload), false); +} + +void WebSocket::QueueOutbound(std::string payload, bool heartbeat) +{ std::scoped_lock lock(m_outboundQueueMutex); + if (m_bShuttingDown || !m_acceptOutbound) + { + if (!heartbeat && !m_bShuttingDown) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[WebSocket] Rejecting message while disconnected; actions are not replayed after reconnect"); + } + return; + } + if (heartbeat && m_heartbeatQueued) + { + return; + } static constexpr size_t kMaxQueuedOutboundMsgs = 256; - if (m_vecQueuedOutboungMsgs.size() >= kMaxQueuedOutboundMsgs) + static constexpr size_t kMaxQueuedOutboundBytes = 2 * 1024 * 1024; + if (m_vecQueuedOutboungMsgs.size() >= kMaxQueuedOutboundMsgs || + payload.size() > kMaxQueuedOutboundBytes - m_queuedOutboundBytes) + { + // the front may be partially transmitted, do not evict it + NetworkLog(ELogVerbosity::LOG_RELEASE, "[WebSocket] Outbound queue full (%zu messages, %zu bytes), rejecting new message", m_vecQueuedOutboungMsgs.size(), m_queuedOutboundBytes); + return; + } + + m_queuedOutboundBytes += payload.size(); + m_vecQueuedOutboungMsgs.push_back({ std::move(payload), heartbeat }); + m_heartbeatQueued |= heartbeat; +} + +void WebSocket::ClearOutbound() +{ + std::scoped_lock lock(m_outboundQueueMutex); + // delivery is ambiguous after a socket failure, so old actions are not replayed on a new socket + if (!m_vecQueuedOutboungMsgs.empty()) { - NetworkLog(ELogVerbosity::LOG_RELEASE, "[WebSocket] Outbound queue full (%zu), discarding oldest message", m_vecQueuedOutboungMsgs.size()); - m_vecQueuedOutboungMsgs.erase(m_vecQueuedOutboungMsgs.begin()); + NetworkLog(ELogVerbosity::LOG_RELEASE, "[WebSocket] Discarding %zu messages from closed connection (%zu bytes, %zu bytes consumed from current message)", m_vecQueuedOutboungMsgs.size(), m_queuedOutboundBytes, m_outboundOffset); } + m_vecQueuedOutboungMsgs.clear(); + m_queuedOutboundBytes = 0; + m_outboundOffset = 0; + m_heartbeatQueued = false; + m_acceptOutbound = false; +} - m_vecQueuedOutboungMsgs.push_back(std::string(send_payload)); +CURLcode WebSocket::FlushOutbound() +{ + // bounded work on the game thread, each call passes the remaining payload of the same frame (curl_ws_send partial writes) + size_t bytesSent = 0; + for (int attempt = 0; attempt < 16; ++attempt) + { + std::scoped_lock lock(m_outboundQueueMutex); + if (m_vecQueuedOutboungMsgs.empty()) + { + return CURLE_OK; + } + const OutboundMessage& message = m_vecQueuedOutboungMsgs.front(); + size_t sent = 0; + const size_t remaining = message.payload.size() - m_outboundOffset; + CURLcode result = curl_ws_send(m_pCurlWS, message.payload.data() + m_outboundOffset, + remaining, &sent, 0, CURLWS_BINARY); + if (sent > remaining) + { + return CURLE_SEND_ERROR; + } + m_outboundOffset += sent; + bytesSent += sent; + if (result != CURLE_OK && result != CURLE_AGAIN) + { + return result; + } + if (m_outboundOffset == message.payload.size()) + { + m_queuedOutboundBytes -= message.payload.size(); + if (message.heartbeat) + { + m_heartbeatQueued = false; + } + m_vecQueuedOutboungMsgs.erase(m_vecQueuedOutboungMsgs.begin()); + m_outboundOffset = 0; + } + if (result == CURLE_AGAIN || sent == 0 || bytesSent >= 256 * 1024) + { + return CURLE_AGAIN; + } + } + return CURLE_AGAIN; } class WebSocketMessageBase @@ -589,6 +657,7 @@ void WebSocket::BeginReconnect() return; } + ClearOutbound(); m_bReconnecting = true; m_numReconnectAttempts = 0; m_bFreshSession = false; @@ -748,6 +817,10 @@ void WebSocket::UpdateReconnect() void WebSocket::Tick() { + if (m_bShuttingDown || m_pMulti == nullptr) + { + return; + } // Main thread only; m_pCurlWS/m_vecWSPartialBuffer/m_bConnected are unlocked here. UpdateReconnect(); @@ -773,13 +846,7 @@ void WebSocket::Tick() pWS->SendData_Signalling(strSignal); */ - // ping? int64_t currTime = NowMs(); - if ((currTime - m_lastPing) > m_timeBetweenUserPings) - { - m_lastPing = currTime; - Ping(); - }; int numReqs = 0; curl_multi_perform(m_pMulti, &numReqs); @@ -845,6 +912,10 @@ void WebSocket::Tick() /* connected and ready */ m_bConnected = true; + { + std::scoped_lock lock(m_outboundQueueMutex); + m_acceptOutbound = true; + } m_vecWSPartialBuffer.clear(); EndReconnect(); @@ -867,40 +938,72 @@ void WebSocket::Tick() return; } - // send anything we have queued (things sent while not connected, or from any other thread) - std::vector outboundBatch; + if ((currTime - m_lastPing) > m_timeBetweenUserPings) { - std::scoped_lock lock(m_outboundQueueMutex); - outboundBatch.swap(m_vecQueuedOutboungMsgs); + m_lastPing = currTime; + Ping(); + } + + const CURLcode sendResult = FlushOutbound(); + if (sendResult != CURLE_OK && sendResult != CURLE_AGAIN) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[WebSocket] Send failed: %s, attempting reconnect", curl_easy_strerror(sendResult)); + m_bConnected = false; + m_vecWSPartialBuffer.clear(); + BeginReconnect(); + return; } - for (std::string& strPayload : outboundBatch) + // drain what the server queued, a signalling burst shouldn't take one frame per tick + static constexpr int MAX_WS_FRAMES_PER_TICK = 64; + CURLcode ret = CURL_LAST; + for (int numFrames = 0; numFrames < MAX_WS_FRAMES_PER_TICK; ++numFrames) { - size_t sent; - CURLcode result = curl_ws_send(m_pCurlWS, strPayload.c_str(), strPayload.length(), &sent, 0, CURLWS_BINARY); + ret = ReceiveOneFrame(); + if (ret != CURLE_OK || !m_bConnected) + { + break; + } + } - if (result != CURLE_OK) + // time since last pong? + if (m_bConnected && m_lastPong != -1 && (currTime - m_lastPong) >= m_timeForWSTimeout) + { + // send event to sentry +#if defined(GENERALS_ONLINE_USE_SENTRY) + if (TheNGMPGame != nullptr) { - NetworkLog(ELogVerbosity::LOG_RELEASE, "curl_ws_send() failed: %s\n", curl_easy_strerror(result)); + AsciiString sentryMsg; + sentryMsg.format("Got websocket disconnect (Timeout: %s), timeout is %lld, last pong was at %lld, current time is %lld, attempting reconnect", curl_easy_strerror(ret), currTime - m_lastPong, m_lastPong, currTime); + sentry_capture_event(sentry_value_new_message_event(SENTRY_LEVEL_ERROR, "WEBSOCKET_DISCONNECT_TIMEOUT", sentryMsg.str())); } - } +#endif + + NetworkLog(ELogVerbosity::LOG_RELEASE, "Got websocket disconnect (Timeout: %s), timeout is %lld, last pong was at %lld, current time is %lld, attempting reconnect", curl_easy_strerror(ret), currTime - m_lastPong, m_lastPong, currTime); + m_bConnected = false; + BeginReconnect(); + m_vecWSPartialBuffer.clear(); + }; +} +// Reads and handles at most one frame, returns the curl_ws_recv result (CURL_LAST if nothing was read) +CURLcode WebSocket::ReceiveOneFrame() +{ // do recv size_t rlen = 0; const struct curl_ws_frame* meta = nullptr; char bufferThisRecv[8196 * 4] = { 0 }; - CURLcode ret = CURL_LAST; - ret = curl_ws_recv(m_pCurlWS, bufferThisRecv, sizeof(bufferThisRecv), &rlen, &meta); + CURLcode ret = curl_ws_recv(m_pCurlWS, bufferThisRecv, sizeof(bufferThisRecv) - 1, &rlen, &meta); // SECURITY FIX: Validate rlen is within buffer bounds - if (rlen > sizeof(bufferThisRecv)) + if (rlen >= sizeof(bufferThisRecv)) { NetworkLog(ELogVerbosity::LOG_RELEASE, "[WebSocket] Received data size %zu exceeds buffer size %zu, discarding", rlen, sizeof(bufferThisRecv)); - return; + return CURL_LAST; } - if (ret != CURLE_RECV_ERROR && ret != CURL_LAST && ret != CURLE_AGAIN && ret != CURLE_GOT_NOTHING) + if (ret == CURLE_OK) { NetworkLog(ELogVerbosity::LOG_DEBUG, "Got websocket msg: %s", bufferThisRecv); NetworkLog(ELogVerbosity::LOG_DEBUG, "Got websocket len: %d", rlen); @@ -925,7 +1028,7 @@ void WebSocket::Tick() { NetworkLog(ELogVerbosity::LOG_RELEASE, "[WebSocket] Partial buffer overflow, discarding message"); m_vecWSPartialBuffer.clear(); - return; + return CURL_LAST; } // SECURITY FIX: Store old size BEFORE resize to avoid off-by-one error in memcpy @@ -1774,7 +1877,7 @@ void WebSocket::Tick() NetworkLog(ELogVerbosity::LOG_DEBUG, "websocket meta was null"); } } - else if (ret == CURLE_RECV_ERROR) + else if (ret != CURLE_AGAIN) { NetworkLog(ELogVerbosity::LOG_RELEASE, "Got websocket disconnect (ERROR: %s), Attempting reconnect", curl_easy_strerror(ret)); @@ -1795,24 +1898,7 @@ void WebSocket::Tick() #endif } - // time since last pong? - if (m_lastPong != -1 && (currTime - m_lastPong) >= m_timeForWSTimeout) - { - // send event to sentry -#if defined(GENERALS_ONLINE_USE_SENTRY) - if (TheNGMPGame != nullptr) - { - AsciiString sentryMsg; - sentryMsg.format("Got websocket disconnect (Timeout: %s), timeout is %lld, last pong was at %lld, current time is %lld, attempting reconnect", curl_easy_strerror(ret), currTime - m_lastPong, m_lastPong, currTime); - sentry_capture_event(sentry_value_new_message_event(SENTRY_LEVEL_ERROR, "WEBSOCKET_DISCONNECT_TIMEOUT", sentryMsg.str())); - } -#endif - - NetworkLog(ELogVerbosity::LOG_RELEASE, "Got websocket disconnect (Timeout: %s), timeout is %lld, last pong was at %lld, current time is %lld, attempting reconnect", curl_easy_strerror(ret), currTime - m_lastPong, m_lastPong, currTime); - m_bConnected = false; - BeginReconnect(); - m_vecWSPartialBuffer.clear(); - }; + return ret; } NGMP_OnlineServices_RoomsInterface::NGMP_OnlineServices_RoomsInterface() From ba712b5deb5818e5008c276e8cc2742ff90676f4 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Thu, 22 Oct 2026 23:13:08 +0200 Subject: [PATCH 16/17] feat(online): Pick a working service route automatically --- GeneralsMD/Code/GameEngine/CMakeLists.txt | 1 + .../GeneralsOnline/GeneralsOnline_Settings.h | 60 ++---- .../GeneralsOnline/HTTP/HTTPManager.h | 10 +- .../GeneralsOnline/HTTP/HTTPRequest.h | 4 +- .../GeneralsOnline/OnlineServices_Auth.h | 1 + .../GeneralsOnline/OnlineServices_Init.h | 14 +- .../GeneralsOnline/ServiceEndpointPolicy.h | 25 +++ .../GeneralsOnline_Settings.cpp | 105 ++++++++-- .../GeneralsOnline/HTTP/HTTPManager.cpp | 57 ++---- .../GeneralsOnline/HTTP/HTTPRequest.cpp | 189 +++++++++--------- .../GeneralsOnline/OnlineServices_Auth.cpp | 26 +++ .../GeneralsOnline/OnlineServices_Init.cpp | 138 +++++++++++-- .../OnlineServices_RoomsInterface.cpp | 125 ++++++------ .../GeneralsOnline/ServiceEndpointPolicy.cpp | 145 ++++++++++++++ 14 files changed, 606 insertions(+), 294 deletions(-) create mode 100644 GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/ServiceEndpointPolicy.h create mode 100644 GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/ServiceEndpointPolicy.cpp diff --git a/GeneralsMD/Code/GameEngine/CMakeLists.txt b/GeneralsMD/Code/GameEngine/CMakeLists.txt index a84c0935e98..b62f340c6fe 100644 --- a/GeneralsMD/Code/GameEngine/CMakeLists.txt +++ b/GeneralsMD/Code/GameEngine/CMakeLists.txt @@ -1163,6 +1163,7 @@ set(GAMEENGINE_SRC Source/GameNetwork/GeneralsOnline/DiscordRichPresence.cpp Source/GameNetwork/GeneralsOnline/NGMP_Helpers.cpp Source/GameNetwork/GeneralsOnline/NGMPGame.cpp + Source/GameNetwork/GeneralsOnline/ServiceEndpointPolicy.cpp Source/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.cpp Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.h index 8ac6d4fb329..94e68938aa8 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.h @@ -1,14 +1,8 @@ #pragma once #include - -enum EHTTPVersion -{ - HTTP_VERSION_AUTO, - HTTP_VERSION_1_0, - HTTP_VERSION_1_1, - HTTP_VERSION_2_0, - HTTP_VERSION_3_0 -}; +#include +#include +#include "GameNetwork/GeneralsOnline/ServiceEndpointPolicy.h" // One layer of data_packs in settings.json: BIG paths in priority order, the first one wins struct GenOnlineDataPackLayer @@ -82,44 +76,13 @@ class GenOnlineSettings const GenOnlineDataPackLayer& DataPacks_CommunityPatch() const { return m_DataPacks_CommunityPatch; } const std::vector& DataPacks_UserExcludePaths() const { return m_DataPacks_UserExcludePaths; } - bool Network_UseAlternativeEndpoint() const { return m_Network_UseAlternativeEndpoint; } - EHTTPVersion Network_GetHTTPVersion() const { return m_Network_HTTPVersion; } - int Network_GetHTTPVersionForCurl() const - { - switch (m_Network_HTTPVersion) - { - case HTTP_VERSION_AUTO: - { - return CURL_HTTP_VERSION_NONE; - } - - case HTTP_VERSION_1_0: - { - return CURL_HTTP_VERSION_1_0; - } - - case HTTP_VERSION_1_1: - { - return CURL_HTTP_VERSION_1_1; - } - - case HTTP_VERSION_2_0: - { - return CURL_HTTP_VERSION_2_0; - } - - case HTTP_VERSION_3_0: - { - return CURL_HTTP_VERSION_3; - } - } - - return CURL_HTTP_VERSION_NONE; - } + ServiceConnectionSettings Network_GetConnectionSettings() const { std::lock_guard lock(m_NetworkMutex); return m_ConnectionSettings; } + std::string Network_GetLastRoute() const { std::lock_guard lock(m_NetworkMutex); return m_LastRoute; } + bool Network_SaveLastRoute(const std::string& host); private: void Load(void); - void Save(); + bool Save(); private: // NOTE: This also works as the default creation (since we just call Save) @@ -157,6 +120,11 @@ class GenOnlineSettings std::string m_Plugins_Anticheat = std::string(); - EHTTPVersion m_Network_HTTPVersion = EHTTPVersion::HTTP_VERSION_AUTO; - bool m_Network_UseAlternativeEndpoint = false; + mutable std::mutex m_NetworkMutex; + ServiceConnectionSettings m_ConnectionSettings; + // host of the service route that worked last + std::string m_LastRoute; + // keeps keys this build does not know when a setting is saved mid-session + std::string m_PreservedJSON = "{}"; + bool m_CanSave = true; }; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h index d2d5c80f753..1b51834ee49 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPManager.h @@ -47,7 +47,7 @@ class HTTPManager return m_sProtocolInUse.load(); } - void AddHandleToMulti(CURL* pNewHandle); + bool AddHandleToMulti(CURL* pNewHandle); void RemoveHandleFromMulti(CURL* pHandleToRemove); void SendGETRequest(const char* szURI, EIPProtocolVersion protover, std::map& inHeaders, std::function completionCallback, std::function progressCallback = nullptr, int timeoutMS = -1); @@ -59,11 +59,6 @@ class HTTPManager void Shutdown(); - bool IsProxyEnabled() const { return m_bProxyEnabled; } - - bool DeterminePlatformProxySettings(); - std::string& GetProxyAddress() { return m_strProxyAddr; } - uint16_t GetProxyPort() const { return m_proxyPort; } private: HTTPRequest* PlatformCreateRequest(EHTTPVerb htpVerb, EIPProtocolVersion protover, const char* szURI, std::map& inHeaders, std::function completionCallback, @@ -74,9 +69,6 @@ class HTTPManager std::atomic m_sProtocolInUse = EIPProtocolVersion::DONT_CARE; - bool m_bProxyEnabled = false; - std::string m_strProxyAddr; - uint16_t m_proxyPort; std::atomic m_bShuttingDown = false; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h index 7dbf977a136..8376c52a71d 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h @@ -75,8 +75,10 @@ class HTTPRequest private: CURL* m_pCURL = nullptr; + bool m_handleAdded = false; - int m_responseCode = -1; + long m_responseCode = 0; + CURLcode m_result = CURLE_OK; EHTTPVerb m_httpVerb; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Auth.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Auth.h index d7c524879e7..45e1d652ee1 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Auth.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Auth.h @@ -23,6 +23,7 @@ class NGMP_OnlineServices_AuthInterface void RefreshToken(); void BeginLogin(); + void BeginLoginOnSelectedRoute(); void DoFullLoginFlow(); void Tick(); diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h index fbe31cc50f2..370e24f16f2 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/OnlineServices_Init.h @@ -227,7 +227,7 @@ class WebSocket private: CURLcode ReceiveOneFrame(); - + void FailConnectionSetup(); CURL* m_pCurlWS = nullptr; CURLM* m_pMulti = nullptr; struct curl_slist* m_pHeaders = nullptr; @@ -540,6 +540,10 @@ class NGMP_OnlineServicesManager } void StartVersionCheck(std::function fnCallback); + void StartVersionCheckOnSelectedRoute(std::function fnCallback); + void EnsureServiceRoute(std::function callback); + static bool IsServiceURLTrusted(const std::string& url); + static bool IsWebSocketURLTrusted(const std::string& url); std::shared_ptr Internal_GetWebSocket() const { return m_pWebSocket; } HTTPManager* GetHTTPManager() const { return m_pHTTPManager; } @@ -654,6 +658,14 @@ class NGMP_OnlineServicesManager std::vector m_vecScreenshotThreads; std::mutex m_mutexScreenshotThreads; + void ProbeServiceRoute(); + void FinishServiceRoute(bool success); + std::string m_serviceBase; + std::vector m_bootstrapRoutes; + size_t m_bootstrapIndex = 0; + std::vector> m_routeCallbacks; + std::chrono::steady_clock::time_point m_routeValidatedAt{}; + ServiceConfig m_ServiceConfig; HTTPManager* m_pHTTPManager = nullptr; diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/ServiceEndpointPolicy.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/ServiceEndpointPolicy.h new file mode 100644 index 00000000000..ba9c44c9709 --- /dev/null +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/ServiceEndpointPolicy.h @@ -0,0 +1,25 @@ +#pragma once +#include +#include +#include + +struct ServiceConnectionSettings +{ + // "automatic" picks a working route + std::string serverMode = "automatic"; +}; + +namespace ServiceConnection +{ +// remembered is the host of the route that worked last, ignored unless it is one of the automatic routes +std::vector BootstrapRoutes(const std::string& environment, + const ServiceConnectionSettings& settings, const std::string& remembered); +// host name of a route url +std::string RouteHost(const std::string& route); +bool IsTrustedServiceURL(const std::string& url, const std::string& base, bool websocket); +// wss address of the websocket on a service base: the scheme swapped from https and ws appended to the path; empty for a base that is not https +std::string WebSocketURLForBase(const std::string& base); +bool IsExpectedVersionManifest(const std::string& body); +// Verified TLS (not in _DEBUG builds), no redirects, no embedded credentials, direct connection; HTTPS or WSS only. +bool ConfigureServiceTransport(CURL* curl, const std::string& url, std::string& error); +} diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.cpp index b8e3ca9bfbc..a3bbea5010c 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.cpp @@ -31,8 +31,12 @@ #define SETTINGS_KEY_DEBUG_VERBOSE_LOGGING "verbose_logging" #define SETTINGS_KEY_NETWORK "network" -#define SETTINGS_KEY_NETWORK_HTTP_VERSION "http_version" +#define SETTINGS_KEY_NETWORK_LEGACY_HTTP_VERSION "http_version" #define SETTINGS_KEY_NETWORK_USE_ALTERNATIVE_ENDPOINT "use_alternative_endpoint" +#define SETTINGS_KEY_NETWORK_SERVER "server" +#define SETTINGS_KEY_NETWORK_SERVER_MODE "mode" +#define SETTINGS_KEY_NETWORK_LAST_ROUTE "last_route" +#define LEGACY_ALTERNATIVE_ROUTE_HOST "api-ru.playgenerals.online" #define SETTINGS_KEY_PLUGINS "plugins" #define SETTINGS_KEY_PLUGINS_ANTICHEAT "anticheat" @@ -253,6 +257,9 @@ void GenOnlineSettings::Load(void) try { jsonSettings = nlohmann::json::parse(strJSON); + if (!jsonSettings.is_object()) + throw std::runtime_error("Settings must be an object"); + m_PreservedJSON = jsonSettings.dump(); } catch (...) @@ -305,28 +312,37 @@ void GenOnlineSettings::Load(void) } } - if (jsonSettings.contains(SETTINGS_KEY_NETWORK)) + if (jsonSettings.contains(SETTINGS_KEY_NETWORK) && jsonSettings[SETTINGS_KEY_NETWORK].is_object()) { - auto networkSettings = jsonSettings[SETTINGS_KEY_NETWORK]; + const auto& networkSettings = jsonSettings[SETTINGS_KEY_NETWORK]; - if (networkSettings.contains(SETTINGS_KEY_NETWORK_HTTP_VERSION)) + if (networkSettings.contains(SETTINGS_KEY_NETWORK_SERVER) && networkSettings[SETTINGS_KEY_NETWORK_SERVER].is_object()) { - int httpVersion = networkSettings[SETTINGS_KEY_NETWORK_HTTP_VERSION]; - - // clamp - if (httpVersion < 0 || httpVersion > HTTP_VERSION_3_0) - { - m_Network_HTTPVersion = EHTTPVersion::HTTP_VERSION_AUTO; - } - else - { - m_Network_HTTPVersion = (EHTTPVersion)httpVersion; - } + const auto& serverSettings = networkSettings[SETTINGS_KEY_NETWORK_SERVER]; + if (serverSettings.contains(SETTINGS_KEY_NETWORK_SERVER_MODE) && serverSettings[SETTINGS_KEY_NETWORK_SERVER_MODE].is_string()) + { + m_ConnectionSettings.serverMode = serverSettings[SETTINGS_KEY_NETWORK_SERVER_MODE].get(); + } + } + if (m_ConnectionSettings.serverMode != "automatic") + { + m_ConnectionSettings.serverMode = "automatic"; } - if (networkSettings.contains(SETTINGS_KEY_NETWORK_USE_ALTERNATIVE_ENDPOINT)) + if (networkSettings.contains(SETTINGS_KEY_NETWORK_LAST_ROUTE) && networkSettings[SETTINGS_KEY_NETWORK_LAST_ROUTE].is_string()) + { + m_LastRoute = networkSettings[SETTINGS_KEY_NETWORK_LAST_ROUTE].get(); + } + else if (networkSettings.contains(SETTINGS_KEY_NETWORK_USE_ALTERNATIVE_ENDPOINT) && + networkSettings[SETTINGS_KEY_NETWORK_USE_ALTERNATIVE_ENDPOINT].is_boolean() && + networkSettings[SETTINGS_KEY_NETWORK_USE_ALTERNATIVE_ENDPOINT].get()) { - m_Network_UseAlternativeEndpoint = networkSettings[SETTINGS_KEY_NETWORK_USE_ALTERNATIVE_ENDPOINT]; + // the old switch from the released client now only prefers that route + m_LastRoute = LEGACY_ALTERNATIVE_ROUTE_HOST; + } + if (m_LastRoute.size() > 253 || m_LastRoute.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-") != std::string::npos) + { + m_LastRoute.clear(); } } @@ -453,6 +469,7 @@ void GenOnlineSettings::Load(void) if (bReadFailed) { + m_CanSave = false; // keep the file on disk, the defaults above only apply to this session NetworkLog(ELogVerbosity::LOG_RELEASE, "[NGMP] Could not read settings file, keeping it untouched"); return; @@ -462,19 +479,30 @@ void GenOnlineSettings::Load(void) { // keep the unparseable file for recovery before the defaults replace it std::filesystem::rename(strSettingsFilePath, strSettingsFilePath + ".bad", ec); + if (ec) + { + m_CanSave = false; + return; + } + m_PreservedJSON = "{}"; } // Always save so we re-serialize anything new or missing Save(); } -void GenOnlineSettings::Save() +bool GenOnlineSettings::Save() { if (!m_bInitialized) { Initialize(); } + if (!m_CanSave) + { + return false; + } + nlohmann::json root = { { SETTINGS_KEY_CAMERA, @@ -497,8 +525,8 @@ void GenOnlineSettings::Save() { SETTINGS_KEY_NETWORK, { - {SETTINGS_KEY_NETWORK_HTTP_VERSION, m_Network_HTTPVersion}, - {SETTINGS_KEY_NETWORK_USE_ALTERNATIVE_ENDPOINT, m_Network_UseAlternativeEndpoint} + {SETTINGS_KEY_NETWORK_SERVER, {{SETTINGS_KEY_NETWORK_SERVER_MODE, m_ConnectionSettings.serverMode}}}, + {SETTINGS_KEY_NETWORK_LAST_ROUTE, m_LastRoute} } }, @@ -547,7 +575,12 @@ void GenOnlineSettings::Save() } }; - std::string strData = root.dump(1); + auto preserved = nlohmann::json::parse(m_PreservedJSON); + preserved.merge_patch(root); + preserved[SETTINGS_KEY_DATAPACKS] = root[SETTINGS_KEY_DATAPACKS]; // the client owns data_packs completely, stale keys go + preserved[SETTINGS_KEY_NETWORK].erase(SETTINGS_KEY_NETWORK_USE_ALTERNATIVE_ENDPOINT); + preserved[SETTINGS_KEY_NETWORK].erase(SETTINGS_KEY_NETWORK_LEGACY_HTTP_VERSION); + std::string strData = preserved.dump(1); std::string strSettingsFilePath = std::format("{}/GeneralsOnlineData/{}", TheGlobalData->getPath_UserData().str(), SETTINGS_FILENAME); @@ -571,4 +604,34 @@ void GenOnlineSettings::Save() std::error_code ec; std::filesystem::remove(strTempPath, ec); } + else + { + m_PreservedJSON = strData; + } + return bSaved; +} + +bool GenOnlineSettings::Network_SaveLastRoute(const std::string& host) +{ + if (host.empty() || host.size() > 253 || host.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-") != std::string::npos) + { + return false; + } + if (!m_bInitialized) + { + Initialize(); + } + std::lock_guard lock(m_NetworkMutex); + if (m_LastRoute == host) + { + return true; + } + const auto previous = m_LastRoute; + m_LastRoute = host; + if (Save()) + { + return true; + } + m_LastRoute = previous; + return false; } diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp index e476ced3bb7..a6a1ea8a3fb 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPManager.cpp @@ -159,44 +159,6 @@ void HTTPManager::Shutdown() } -bool HTTPManager::DeterminePlatformProxySettings() -{ - CHECK_MAIN_THREAD; - - WINHTTP_CURRENT_USER_IE_PROXY_CONFIG pProxyConfig; - WinHttpGetIEProxyConfigForCurrentUser(&pProxyConfig); - - if (pProxyConfig.lpszProxy != nullptr) - { - LPWSTR ws = pProxyConfig.lpszProxy; - std::string strFullProxy; - strFullProxy.reserve(wcslen(ws)); - for (; *ws; ws++) - strFullProxy += (char)*ws; - - size_t ipStart = strFullProxy.find("="); - ipStart = (ipStart != std::string::npos) ? ipStart + 1 : 0; - size_t ipEnd = strFullProxy.find(":", ipStart); - if (ipEnd == std::string::npos) ipEnd = strFullProxy.size(); - - m_strProxyAddr = strFullProxy.substr(ipStart, ipEnd - ipStart); - - size_t portStart = ipEnd + 1; - size_t portEnd = strFullProxy.find(";", portStart); - std::string strPort = strFullProxy.substr(portStart, portEnd != std::string::npos ? portEnd - portStart : std::string::npos); - - m_proxyPort = (uint16_t)atoi(strPort.c_str()); - } - - m_bProxyEnabled = pProxyConfig.lpszProxy != nullptr; - - if (pProxyConfig.lpszProxy) GlobalFree(pProxyConfig.lpszProxy); - if (pProxyConfig.lpszAutoConfigUrl) GlobalFree(pProxyConfig.lpszAutoConfigUrl); - if (pProxyConfig.lpszProxyBypass) GlobalFree(pProxyConfig.lpszProxyBypass); - - return m_bProxyEnabled; -} - HTTPRequest* HTTPManager::PlatformCreateRequest(EHTTPVerb httpVerb, EIPProtocolVersion protover, const char* szURI, std::map& inHeaders, std::function completionCallback, std::function progressCallback /*= nullptr*/, int timeoutMS /* = -1 */) noexcept { CHECK_MAIN_THREAD; @@ -217,7 +179,6 @@ void HTTPManager::Initialize() CHECK_MAIN_THREAD; m_pCurl = curl_multi_init(); - m_bProxyEnabled = DeterminePlatformProxySettings(); } void HTTPManager::Tick() @@ -226,13 +187,19 @@ void HTTPManager::Tick() std::vector vecItemsToRemove = std::vector(); - // start anything needing starting - for (HTTPRequest* pRequest : m_vecRequestsPendingStart) + // callbacks may enqueue new requests, so do not iterate the live pending vector + auto pending = std::move(m_vecRequestsPendingStart); + m_vecRequestsPendingStart.clear(); + for (HTTPRequest* pRequest : pending) { pRequest->StartRequest(); m_vecRequestsInFlight.push_back(pRequest); + // a request refused before reaching curl completes synchronously + if (pRequest->IsComplete()) + { + vecItemsToRemove.push_back(pRequest); + } } - m_vecRequestsPendingStart.clear(); // perform and poll if (m_pCurl == nullptr) @@ -289,16 +256,16 @@ void HTTPManager::Tick() // remove any completed for (HTTPRequest* pRequestToDestroy : vecItemsToRemove) { - m_vecRequestsInFlight.erase(std::remove(m_vecRequestsInFlight.begin(), m_vecRequestsInFlight.end(), pRequestToDestroy)); + m_vecRequestsInFlight.erase(std::remove(m_vecRequestsInFlight.begin(), m_vecRequestsInFlight.end(), pRequestToDestroy), m_vecRequestsInFlight.end()); delete pRequestToDestroy; } } -void HTTPManager::AddHandleToMulti(CURL* pNewHandle) +bool HTTPManager::AddHandleToMulti(CURL* pNewHandle) { CHECK_MAIN_THREAD; - curl_multi_add_handle(m_pCurl, pNewHandle); + return m_pCurl != nullptr && curl_multi_add_handle(m_pCurl, pNewHandle) == CURLM_OK; } void HTTPManager::RemoveHandleFromMulti(CURL* pHandleToRemove) diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp index 3c77e8f4039..2fc21306ccd 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp @@ -1,4 +1,5 @@ #include "GameNetwork/GeneralsOnline/HTTP/HTTPRequest.h" +#include "GameNetwork/GeneralsOnline/ServiceEndpointPolicy.h" #include "GameNetwork/GeneralsOnline/OnlineServices_Init.h" #include "GameNetwork/GeneralsOnline/HTTP/HTTPManager.h" #include "GameNetwork/GeneralsOnline/NGMP_interfaces.h" @@ -35,22 +36,13 @@ HTTPRequest::HTTPRequest(EHTTPVerb httpVerb, EIPProtocolVersion protover, const HTTPRequest::~HTTPRequest() { - NGMP_OnlineServicesManager* pMgr = NGMP_OnlineServicesManager::GetInstance(); - if (pMgr == nullptr) - return; - - HTTPManager* pHTTPManager = pMgr->GetHTTPManager(); - pHTTPManager->RemoveHandleFromMulti(m_pCURL); - - curl_easy_cleanup(m_pCURL); - - m_vecBuffer.clear(); - - if (headers) + NGMP_OnlineServicesManager* manager = NGMP_OnlineServicesManager::GetInstance(); + if (m_handleAdded && manager != nullptr && manager->GetHTTPManager() != nullptr) { - curl_slist_free_all(headers); - headers = nullptr; - } + manager->GetHTTPManager()->RemoveHandleFromMulti(m_pCURL); + } + curl_easy_cleanup(m_pCURL); + curl_slist_free_all(headers); } @@ -119,7 +111,7 @@ void HTTPRequest::InvokeCallbackIfComplete() { strResponse.clear(); } - m_completionCallback(true, m_responseCode, strResponse, this); + m_completionCallback(m_result == CURLE_OK, static_cast(m_responseCode), strResponse, this); } } } @@ -152,8 +144,9 @@ void HTTPRequest::Threaded_SetComplete(CURLcode result) { // store response code curl_easy_getinfo(m_pCURL, CURLINFO_RESPONSE_CODE, &m_responseCode); + m_result = result; - if (result == CURLE_OK) + if (result == CURLE_OK && NGMP_OnlineServicesManager::IsServiceURLTrusted(m_strURI)) { HTTPManager* pHTTPManager = static_cast(NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()); if (pHTTPManager != nullptr) @@ -233,104 +226,106 @@ void HTTPRequest::Threaded_SetComplete(CURLcode result) void HTTPRequest::PlatformStartRequest() { - if (m_pCURL) + if (m_pCURL == nullptr) { - HTTPManager* pHTTPManager = static_cast(NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()); - - curl_easy_setopt(m_pCURL, CURLOPT_URL, m_strURI.c_str()); - curl_easy_setopt(m_pCURL, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(m_pCURL, CURLOPT_WRITEDATA, (void*)this); - curl_easy_setopt(m_pCURL, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); - curl_easy_setopt(m_pCURL, CURLOPT_USERAGENT, "GeneralsOnline Client"); - - - curl_easy_setopt(m_pCURL, CURLOPT_CONNECTTIMEOUT_MS, m_timeoutMS); - curl_easy_setopt(m_pCURL, CURLOPT_TIMEOUT_MS, m_timeoutMS); - - curl_easy_setopt(m_pCURL, CURLOPT_HTTP_VERSION, NGMP_OnlineServicesManager::Settings.Network_GetHTTPVersionForCurl()); - - if (m_protover == EIPProtocolVersion::DONT_CARE) - { - curl_easy_setopt(m_pCURL, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_WHATEVER); - } - else if (m_protover == EIPProtocolVersion::FORCE_IPV4) - { - curl_easy_setopt(m_pCURL, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); - } - else if (m_protover == EIPProtocolVersion::FORCE_IPV6) - { - curl_easy_setopt(m_pCURL, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); - } - + Threaded_SetComplete(CURLE_FAILED_INIT); + return; + } + std::string error; + if (!ServiceConnection::ConfigureServiceTransport(m_pCURL, m_strURI, error)) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[HTTP] Connection policy: %s", error.c_str()); + Threaded_SetComplete(CURLE_FAILED_INIT); + return; + } - // Are we authenticated? attach our auth header - NGMP_OnlineServices_AuthInterface* pAuthInterface = NGMP_OnlineServicesManager::GetInterface(); - if (pAuthInterface != nullptr && pAuthInterface->IsLoggedIn()) + // credentials only ever go to the selected service route + const bool trusted = NGMP_OnlineServicesManager::IsServiceURLTrusted(m_strURI); + for (const auto& kvPair : m_mapHeaders) + { + if (_stricmp(kvPair.first.c_str(), "Authorization") == 0 && !trusted) { - if (m_bAppendAuthIfPresent) - { - m_mapHeaders["Authorization"] = "Bearer " + pAuthInterface->GetAuthToken(); - } + NetworkLog(ELogVerbosity::LOG_RELEASE, "[HTTP] Refused credentials outside selected service route"); + Threaded_SetComplete(CURLE_URL_MALFORMAT); + return; } + } - for (auto& kvPair : m_mapHeaders) - { - std::string strHeader = kvPair.first + ": " + kvPair.second; - headers = curl_slist_append(headers, strHeader.c_str()); - } - curl_easy_setopt(m_pCURL, CURLOPT_HTTPHEADER, headers); + // Are we authenticated? attach our auth header + NGMP_OnlineServices_AuthInterface* pAuthInterface = NGMP_OnlineServicesManager::GetInterface(); + if (trusted && m_bAppendAuthIfPresent && pAuthInterface != nullptr && pAuthInterface->IsLoggedIn()) + { + m_mapHeaders["Authorization"] = "Bearer " + pAuthInterface->GetAuthToken(); + } - if (m_httpVerb == EHTTPVerb::HTTP_VERB_POST || m_httpVerb == EHTTPVerb::HTTP_VERB_PUT || m_httpVerb == EHTTPVerb::HTTP_VERB_DELETE) + for (const auto& kvPair : m_mapHeaders) + { + const std::string strHeader = kvPair.first + ": " + kvPair.second; + curl_slist* updated = curl_slist_append(headers, strHeader.c_str()); + if (updated == nullptr) { - //if (m_strPostData.length() > 0) - { - //char* pEscaped = curl_easy_escape(m_pCURL, m_strPostData.c_str(), m_strPostData.length()); - - if (!m_vecPostDataBuffer.empty()) - { - curl_easy_setopt(m_pCURL, CURLOPT_POSTFIELDS, m_vecPostDataBuffer.data()); - curl_easy_setopt(m_pCURL, CURLOPT_POSTFIELDSIZE, m_vecPostDataBuffer.size()); - } - else - { - curl_easy_setopt(m_pCURL, CURLOPT_POSTFIELDS, m_strPostData.c_str()); - } - } + Threaded_SetComplete(CURLE_OUT_OF_MEMORY); + return; } + headers = updated; + } - if (m_maxSendBytesPerSec > 0) + CURLcode result = CURLE_OK; + auto option = [&](CURLoption key, auto value) { + if (result == CURLE_OK) { - curl_easy_setopt(m_pCURL, CURLOPT_MAX_SEND_SPEED_LARGE, (curl_off_t)m_maxSendBytesPerSec); + result = curl_easy_setopt(m_pCURL, key, value); } - - // needed for PUT etc - if (m_httpVerb == EHTTPVerb::HTTP_VERB_PUT) + }; + option(CURLOPT_URL, m_strURI.c_str()); + option(CURLOPT_WRITEDATA, static_cast(this)); + option(CURLOPT_WRITEFUNCTION, WriteMemoryCallback); + option(CURLOPT_USERAGENT, "GeneralsOnline Client"); + option(CURLOPT_CONNECTTIMEOUT_MS, static_cast(m_timeoutMS)); + option(CURLOPT_TIMEOUT_MS, static_cast(m_timeoutMS)); + option(CURLOPT_HTTP_VERSION, static_cast(CURL_HTTP_VERSION_NONE)); + option(CURLOPT_IPRESOLVE, static_cast(m_protover == EIPProtocolVersion::FORCE_IPV4 ? CURL_IPRESOLVE_V4 : + m_protover == EIPProtocolVersion::FORCE_IPV6 ? CURL_IPRESOLVE_V6 : CURL_IPRESOLVE_WHATEVER)); + option(CURLOPT_HTTPHEADER, headers); + + if (m_httpVerb == EHTTPVerb::HTTP_VERB_POST || m_httpVerb == EHTTPVerb::HTTP_VERB_PUT || m_httpVerb == EHTTPVerb::HTTP_VERB_DELETE) + { + if (!m_vecPostDataBuffer.empty()) { - curl_easy_setopt(m_pCURL, CURLOPT_CUSTOMREQUEST, "PUT"); + option(CURLOPT_POSTFIELDS, m_vecPostDataBuffer.data()); + option(CURLOPT_POSTFIELDSIZE_LARGE, static_cast(m_vecPostDataBuffer.size())); } - else if (m_httpVerb == EHTTPVerb::HTTP_VERB_DELETE) + else { - curl_easy_setopt(m_pCURL, CURLOPT_CUSTOMREQUEST, "DELETE"); + option(CURLOPT_POSTFIELDS, m_strPostData.c_str()); + option(CURLOPT_POSTFIELDSIZE_LARGE, static_cast(m_strPostData.size())); } + } -#if _DEBUG - if (pHTTPManager->IsProxyEnabled()) - { - curl_easy_setopt(m_pCURL, CURLOPT_PROXY, pHTTPManager->GetProxyAddress().c_str()); - curl_easy_setopt(m_pCURL, CURLOPT_PROXYPORT, pHTTPManager->GetProxyPort()); - } + if (m_maxSendBytesPerSec > 0) + { + option(CURLOPT_MAX_SEND_SPEED_LARGE, static_cast(m_maxSendBytesPerSec)); + } - curl_easy_setopt(m_pCURL, CURLOPT_SSL_VERIFYPEER, 0); - curl_easy_setopt(m_pCURL, CURLOPT_SSL_VERIFYHOST, 0); - curl_easy_setopt(m_pCURL, CURLOPT_VERBOSE, 1); -#else + // needed for PUT etc + if (m_httpVerb == EHTTPVerb::HTTP_VERB_PUT) + { + option(CURLOPT_CUSTOMREQUEST, "PUT"); + } + else if (m_httpVerb == EHTTPVerb::HTTP_VERB_DELETE) + { + option(CURLOPT_CUSTOMREQUEST, "DELETE"); + } - // Use the OS cert store (needed since we build curl against OpenSSL, not schannel). - curl_easy_setopt(m_pCURL, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA); - curl_easy_setopt(m_pCURL, CURLOPT_SSL_VERIFYPEER, 1L); - curl_easy_setopt(m_pCURL, CURLOPT_SSL_VERIFYHOST, 2L); -#endif + if (result != CURLE_OK) + { + Threaded_SetComplete(result); + return; + } - pHTTPManager->AddHandleToMulti(m_pCURL); + m_handleAdded = NGMP_OnlineServicesManager::GetInstance()->GetHTTPManager()->AddHandleToMulti(m_pCURL); + if (!m_handleAdded) + { + Threaded_SetComplete(CURLE_FAILED_INIT); } } diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Auth.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Auth.cpp index 4bdfb583a4b..40f85720725 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Auth.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Auth.cpp @@ -302,6 +302,32 @@ void NGMP_OnlineServices_AuthInterface::RefreshToken() } void NGMP_OnlineServices_AuthInterface::BeginLogin() +{ + auto* manager = NGMP_OnlineServicesManager::GetInstance(); + if (!manager) + { + return; + } + manager->EnsureServiceRoute([](bool success) + { + auto* auth = NGMP_OnlineServicesManager::GetInterface(); + if (!auth) + { + return; + } + if (success) + { + auth->BeginLoginOnSelectedRoute(); + } + else + { + ClearGSMessageBoxes(); + GSMessageBoxOk(UnicodeString(L"Connection failed"), UnicodeString(L"No trusted service route could be reached. Check your connection and try again."), []() { TheShell->pop(); }); + } + }); +} + +void NGMP_OnlineServices_AuthInterface::BeginLoginOnSelectedRoute() { m_tokenCreationTime = -1; diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp index b2fa378fdb1..b253dbdc003 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Init.cpp @@ -15,8 +15,10 @@ #include "GameClient/Display.h" #include "WW3D2/surfaceclass.h" #include "WW3D2/dx8wrapper.h" +#include #include #include +#include "GameNetwork/GeneralsOnline/ServiceEndpointPolicy.h" #define STB_IMAGE_WRITE_IMPLEMENTATION #define STB_IMAGE_RESIZE_IMPLEMENTATION @@ -206,33 +208,116 @@ GenOnlineSettings NGMP_OnlineServicesManager::Settings; NGMP_OnlineServicesManager::NGMP_OnlineServicesManager() { NetworkLog(ELogVerbosity::LOG_RELEASE, "[NGMP] Init"); +#if _DEBUG + NetworkLog(ELogVerbosity::LOG_RELEASE, "[NGMP] WARNING: TLS verification is disabled in debug builds"); +#endif m_pOnlineServicesManager = this; } +static const char* ServiceEnvironment() +{ + switch (NGMP_OnlineServicesManager::g_Environment) + { + case NGMP_OnlineServicesManager::EEnvironment::DEV: return "dev"; + case NGMP_OnlineServicesManager::EEnvironment::TEST: return "test"; + default: return "prod"; + } +} + std::string NGMP_OnlineServicesManager::GetAPIEndpoint(const char* szEndpoint) { - if (g_Environment == EEnvironment::DEV) + auto* manager = GetInstance(); + if (manager && !manager->m_serviceBase.empty()) { - return std::format("https://localhost:9000/env/dev/contract/1/{}", szEndpoint); + return manager->m_serviceBase + szEndpoint; } - else if (g_Environment == EEnvironment::TEST) + const auto routes = ServiceConnection::BootstrapRoutes(ServiceEnvironment(), Settings.Network_GetConnectionSettings(), Settings.Network_GetLastRoute()); + return routes.empty() ? std::string() : routes.front() + szEndpoint; +} + +bool NGMP_OnlineServicesManager::IsServiceURLTrusted(const std::string& url) +{ + return ServiceConnection::IsTrustedServiceURL(url, GetAPIEndpoint(""), false); +} + +bool NGMP_OnlineServicesManager::IsWebSocketURLTrusted(const std::string& url) +{ + return ServiceConnection::IsTrustedServiceURL(url, GetAPIEndpoint(""), true); +} + +void NGMP_OnlineServicesManager::EnsureServiceRoute(std::function callback) +{ + // never change the route of an authenticated session behind its caller + if (m_pAuthInterface && m_pAuthInterface->IsLoggedIn()) { - return std::format("https://api.playgenerals.online:2087/env/test/contract/1/{}", szEndpoint); + callback(!m_serviceBase.empty()); + return; } - else // PROD + if (!m_serviceBase.empty() && std::chrono::steady_clock::now() - m_routeValidatedAt < std::chrono::seconds(30)) { - if (NGMP_OnlineServicesManager::Settings.Network_UseAlternativeEndpoint()) - { - return std::format("https://api-ru.playgenerals.online/env/prod/contract/1/{}", szEndpoint); - } - else - { - return std::format("https://api.playgenerals.online/env/prod/contract/1/{}", szEndpoint); - } + callback(true); + return; + } + m_routeCallbacks.push_back(std::move(callback)); + if (m_routeCallbacks.size() > 1) + { + return; + } + m_bootstrapRoutes = ServiceConnection::BootstrapRoutes(ServiceEnvironment(), Settings.Network_GetConnectionSettings(), Settings.Network_GetLastRoute()); + m_bootstrapIndex = 0; + ProbeServiceRoute(); +} + +void NGMP_OnlineServicesManager::FinishServiceRoute(bool success) +{ + auto callbacks = std::move(m_routeCallbacks); + m_routeCallbacks.clear(); + for (auto& callback : callbacks) + { + callback(success); } } +void NGMP_OnlineServicesManager::ProbeServiceRoute() +{ + if (m_bootstrapIndex >= m_bootstrapRoutes.size()) + { + m_serviceBase.clear(); + FinishServiceRoute(false); + return; + } + + // anonymous request: no credentials are sent until a route is validated + m_serviceBase = m_bootstrapRoutes[m_bootstrapIndex]; + const std::string url = m_serviceBase + "VersionManifest"; + std::map headers; + m_pHTTPManager->SendGETRequest(url.c_str(), EIPProtocolVersion::DONT_CARE, headers, + [](bool success, int status, std::string body, HTTPRequest*) + { + auto* manager = GetInstance(); + if (!manager) + { + return; + } + if (success && status == 200 && ServiceConnection::IsExpectedVersionManifest(body)) + { + manager->m_routeValidatedAt = std::chrono::steady_clock::now(); + if (g_Environment == EEnvironment::PROD) + { + Settings.Network_SaveLastRoute(ServiceConnection::RouteHost(manager->m_serviceBase)); + } + NetworkLog(ELogVerbosity::LOG_RELEASE, "[Connection] Trusted service route validated: %s", manager->m_serviceBase.c_str()); + manager->FinishServiceRoute(true); + return; + } + NetworkLog(ELogVerbosity::LOG_RELEASE, "[Connection] Route validation failed (HTTP %d); credentials were not sent", status); + // an access rejection is not evidence that another route would work + manager->m_bootstrapIndex = (status == 401 || status == 403) ? manager->m_bootstrapRoutes.size() : manager->m_bootstrapIndex + 1; + manager->ProbeServiceRoute(); + }, nullptr, 6000); +} + void NGMP_OnlineServicesManager::AttemptLoadSteam() { // app id for ZH @@ -408,6 +493,22 @@ void NGMP_OnlineServicesManager::Shutdown() } void NGMP_OnlineServicesManager::StartVersionCheck(std::function fnCallback) +{ + EnsureServiceRoute([fnCallback](bool success) + { + auto* manager = GetInstance(); + if (!manager) + return; + if (!success) + { + fnCallback(false, false); + return; + } + manager->StartVersionCheckOnSelectedRoute(fnCallback); + }); +} + +void NGMP_OnlineServicesManager::StartVersionCheckOnSelectedRoute(std::function fnCallback) { std::string strURI = NGMP_OnlineServicesManager::GetAPIEndpoint("VersionCheck"); @@ -865,8 +966,15 @@ void NGMP_OnlineServicesManager::OnLogin(ELoginResult loginResult, const char* s // connect to WS m_pWebSocket = std::make_shared(); - // TODO_NGMP: This should come from the service, if the service was russia-aware - std::string strWebsocketAddr = NGMP_OnlineServicesManager::Settings.Network_UseAlternativeEndpoint() ? "wss://api-ru.playgenerals.online/ws" : std::string(szWSAddr); + // the websocket address comes from the route we logged in through, not from the service + const std::string strWebsocketAddr = ServiceConnection::WebSocketURLForBase(GetAPIEndpoint("")); + const std::string strServiceAddr = szWSAddr ? szWSAddr : ""; + if (strServiceAddr != strWebsocketAddr) + { + const std::string strServiceHost = ServiceConnection::RouteHost(strServiceAddr); + const bool bPrintable = std::none_of(strServiceHost.begin(), strServiceHost.end(), [](unsigned char c) { return c <= 32 || c >= 127; }); + NetworkLog(ELogVerbosity::LOG_RELEASE, "[Connection] Service websocket address (host %s) differs from the route, using %s", bPrintable ? strServiceHost.c_str() : "[invalid]", strWebsocketAddr.c_str()); + } m_pWebSocket->Connect(strWebsocketAddr.c_str(), false, [=]() { diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp index bf388eecd86..b91f9c9df63 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp @@ -1,4 +1,5 @@ #include +#include "GameNetwork/GeneralsOnline/ServiceEndpointPolicy.h" #include "GameNetwork/GeneralsOnline/NGMP_interfaces.h" #include "GameNetwork/GeneralsOnline/NGMP_include.h" #include "GameNetwork/GeneralsOnline/NetworkPacket.h" @@ -152,86 +153,92 @@ int WebSocket::Ping() void WebSocket::Connect(const char* url, bool bIsReconnect, std::function fnWebsocketConnectedCallback) { - if (m_bConnected) + if (m_bConnected || m_bShuttingDown) { return; } m_lastPong = NowMs(); + // copy before replacing the stored address: reconnect passes its c_str() + const std::string destination(url); + m_strWebsocketAddr = destination; + m_fnWebsocketConnectedCallback = fnWebsocketConnectedCallback; + // TODO_CACHE: Cleanup multi too if (m_pCurlWS != nullptr) { - // remove from multi before cleanup (required by libcurl) - if (m_pMulti != nullptr) - { - curl_multi_remove_handle(m_pMulti, m_pCurlWS); - } - // cleanup - curl_easy_cleanup(m_pCurlWS); - m_pCurlWS = nullptr; + // remove from multi before cleanup (required by libcurl) + if (m_pMulti != nullptr) + { + curl_multi_remove_handle(m_pMulti, m_pCurlWS); + } + curl_easy_cleanup(m_pCurlWS); + m_pCurlWS = nullptr; } - // Free old headers before creating new ones - if (m_pHeaders != nullptr) + // Free old headers before creating new ones + curl_slist_free_all(m_pHeaders); + m_pHeaders = nullptr; + + // the token only ever goes to the selected service route + NGMP_OnlineServices_AuthInterface* pAuthInterface = NGMP_OnlineServicesManager::GetInterface(); + if (pAuthInterface == nullptr || !NGMP_OnlineServicesManager::IsWebSocketURLTrusted(destination)) { - curl_slist_free_all(m_pHeaders); - m_pHeaders = nullptr; + NetworkLog(ELogVerbosity::LOG_RELEASE, "[WebSocket] Refused untrusted service destination"); + FailConnectionSetup(); + return; } m_pCurlWS = curl_easy_init(); - - if (m_pCurlWS != nullptr) + std::string error; + if (m_pCurlWS == nullptr || !ServiceConnection::ConfigureServiceTransport(m_pCurlWS, destination, error)) { - m_fnWebsocketConnectedCallback = fnWebsocketConnectedCallback; - - int httpResponseCode = -1; - m_strWebsocketAddr = std::string(url); - curl_easy_setopt(m_pCurlWS, CURLOPT_URL, url); - - curl_easy_getinfo(m_pCurlWS, CURLINFO_RESPONSE_CODE, &httpResponseCode); - - curl_easy_setopt(m_pCurlWS, CURLOPT_CONNECT_ONLY, 2L); /* websocket style */ - - // HTTP v1 seems to have a higher success rate of bypassing DPI - curl_easy_setopt(m_pCurlWS, CURLOPT_HTTP_VERSION, NGMP_OnlineServicesManager::Settings.Network_GetHTTPVersionForCurl()); - -#if _DEBUG - curl_easy_setopt(m_pCurlWS, CURLOPT_SSL_VERIFYPEER, 0); - curl_easy_setopt(m_pCurlWS, CURLOPT_SSL_VERIFYHOST, 0); - - curl_easy_setopt(m_pCurlWS, CURLOPT_VERBOSE, 1L); -#else - // Use the OS cert store (needed since we build curl against OpenSSL, not schannel). - curl_easy_setopt(m_pCurlWS, CURLOPT_SSL_OPTIONS, CURLSSLOPT_NATIVE_CA); - curl_easy_setopt(m_pCurlWS, CURLOPT_SSL_VERIFYPEER, 1L); - curl_easy_setopt(m_pCurlWS, CURLOPT_SSL_VERIFYHOST, 2L); -#endif + NetworkLog(ELogVerbosity::LOG_RELEASE, "[WebSocket] Connection policy: %s", error.c_str()); + FailConnectionSetup(); + return; + } + const std::string authorization = "Authorization: Bearer " + pAuthInterface->GetAuthToken(); + m_pHeaders = curl_slist_append(nullptr, authorization.c_str()); + curl_slist* updated = m_pHeaders ? curl_slist_append(m_pHeaders, bIsReconnect ? "is-reconnect: true" : "is-reconnect: false") : nullptr; + if (updated == nullptr) + { + FailConnectionSetup(); + return; + } + m_pHeaders = updated; - // ws needs auth - NGMP_OnlineServices_AuthInterface* pAuthInterface = NGMP_OnlineServicesManager::GetInterface(); - if (pAuthInterface == nullptr) + CURLcode result = CURLE_OK; + auto option = [&](CURLoption key, auto value) { + if (result == CURLE_OK) { - curl_easy_cleanup(m_pCurlWS); - m_pCurlWS = nullptr; - return; + result = curl_easy_setopt(m_pCurlWS, key, value); } + }; + option(CURLOPT_URL, destination.c_str()); + option(CURLOPT_CONNECT_ONLY, 2L); /* websocket style */ + // HTTP v1 seems to have a higher success rate of bypassing DPI + option(CURLOPT_HTTP_VERSION, static_cast(CURL_HTTP_VERSION_NONE)); + option(CURLOPT_HTTPHEADER, m_pHeaders); + if (result != CURLE_OK || curl_multi_add_handle(m_pMulti, m_pCurlWS) != CURLM_OK) + { + FailConnectionSetup(); + } +} - char szHeaderBuffer[8192] = { 0 }; - sprintf_s(szHeaderBuffer, "Authorization: Bearer %s", pAuthInterface->GetAuthToken().c_str()); - m_pHeaders = curl_slist_append(m_pHeaders, szHeaderBuffer); - - sprintf_s(szHeaderBuffer, "is-reconnect: %s", bIsReconnect ? "true": "false"); - m_pHeaders = curl_slist_append(m_pHeaders, szHeaderBuffer); - - curl_easy_setopt(m_pCurlWS, CURLOPT_HTTPHEADER, m_pHeaders); - - //curl_easy_setopt(m_pCurl, CURLOPT_TIMEOUT_MS, 1000); - - /* Perform the request, res gets the return code */ - //CURLcode res = curl_easy_perform(m_pCurl); - curl_multi_add_handle(m_pMulti, m_pCurlWS); +void WebSocket::FailConnectionSetup() +{ + m_bConnected = false; + m_vecWSPartialBuffer.clear(); + if (m_bReconnecting) + { + m_reconnectAttemptStarted = -1; + m_nextReconnectAttempt = NowMs() + ComputeReconnectDelay(); + } + else + { + NGMP_OnlineServicesManager::GetInstance()->SetPendingFullTeardown(EGOTearDownReason::LOST_CONNECTION); } } diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/ServiceEndpointPolicy.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/ServiceEndpointPolicy.cpp new file mode 100644 index 00000000000..f94c5070ce5 --- /dev/null +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/ServiceEndpointPolicy.cpp @@ -0,0 +1,145 @@ +#include "GameNetwork/GeneralsOnline/ServiceEndpointPolicy.h" +#include "GameNetwork/GeneralsOnline/json.hpp" +#include +#include +#include + +namespace ServiceConnection +{ +std::vector BootstrapRoutes(const std::string& environment, + const ServiceConnectionSettings& settings, const std::string& remembered) +{ + if (environment == "dev") + return {"https://localhost:9000/env/dev/contract/1/"}; + if (environment == "test") + return {"https://api.playgenerals.online:2087/env/test/contract/1/"}; + if (environment != "prod") + return {}; + if (settings.serverMode != "automatic") + return {}; + std::vector routes = { + "https://api.playgenerals.online/env/prod/contract/1/", + "https://api-ru.playgenerals.online/env/prod/contract/1/", + }; + const auto remembered_route = std::find_if(routes.begin(), routes.end(), [&](const std::string& route) { return RouteHost(route) == remembered; }); + if (remembered_route != routes.end()) + std::rotate(routes.begin(), remembered_route, remembered_route + 1); + return routes; +} + +std::string RouteHost(const std::string& route) +{ + const auto schemeEnd = route.find("://"); + if (schemeEnd == std::string::npos) + return {}; + const auto start = schemeEnd + 3; + return route.substr(start, route.find_first_of(":/", start) - start); +} + +static std::string Lower(std::string value) +{ + std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) { return static_cast(tolower(c)); }); + return value; +} + +static bool EndsWith(const std::string& value, const std::string& suffix) +{ + return value.size() >= suffix.size() && value.compare(value.size() - suffix.size(), suffix.size(), suffix) == 0; +} + +static std::string Part(CURLU* url, CURLUPart part) +{ + char* value = nullptr; + if (curl_url_get(url, part, &value, CURLU_DEFAULT_PORT) != CURLUE_OK) + return {}; + std::string result(value); + curl_free(value); + return result; +} + +bool IsTrustedServiceURL(const std::string& value, const std::string& base, bool websocket) +{ + if (value.find_first_of("\\\r\n\t #") != std::string::npos) + return false; + const auto schemeEnd = value.find("://"); + if (schemeEnd == std::string::npos) + return false; + const auto authorityEnd = value.find_first_of("/?", schemeEnd + 3); + const auto authority = value.substr(schemeEnd + 3, + authorityEnd == std::string::npos ? std::string::npos : authorityEnd - schemeEnd - 3); + if (authority.find('@') != std::string::npos || (websocket && value.find('?') != std::string::npos)) + return false; + std::string rawPath; + if (authorityEnd != std::string::npos && value[authorityEnd] == '/') + rawPath = value.substr(authorityEnd, value.find('?', authorityEnd) - authorityEnd); + // query escaping is legitimate; path separators and dot segments must not change after validation + const auto lowerPath = Lower(rawPath); + if ((websocket && rawPath.find('%') != std::string::npos) || lowerPath.find("%2e") != std::string::npos || + lowerPath.find("%2f") != std::string::npos || lowerPath.find("%5c") != std::string::npos || + rawPath.find("/../") != std::string::npos || rawPath.find("/./") != std::string::npos || + EndsWith(rawPath, "/..") || EndsWith(rawPath, "/.")) + return false; + std::unique_ptr url(curl_url(), curl_url_cleanup); + std::unique_ptr expected(curl_url(), curl_url_cleanup); + if (!url || !expected || curl_url_set(url.get(), CURLUPART_URL, value.c_str(), CURLU_NON_SUPPORT_SCHEME) != CURLUE_OK || + curl_url_set(expected.get(), CURLUPART_URL, base.c_str(), 0) != CURLUE_OK) + return false; + if (Part(url.get(), CURLUPART_SCHEME) != (websocket ? "wss" : "https")) + return false; + auto port = Part(url.get(), CURLUPART_PORT); + if (port.empty() && websocket) + port = "443"; + if (Lower(Part(url.get(), CURLUPART_HOST)) != Lower(Part(expected.get(), CURLUPART_HOST)) || + port != Part(expected.get(), CURLUPART_PORT)) + return false; + const auto path = Part(url.get(), CURLUPART_PATH); + const auto expectedPath = Part(expected.get(), CURLUPART_PATH); + if (!websocket) + return path.compare(0, expectedPath.size(), expectedPath) == 0; + return EndsWith(path, expectedPath + "ws"); +} + +std::string WebSocketURLForBase(const std::string& base) +{ + if (base.compare(0, 8, "https://") != 0) + return {}; + return "wss://" + base.substr(8) + "ws"; +} + +bool IsExpectedVersionManifest(const std::string& body) +{ + const auto json = nlohmann::json::parse(body, nullptr, false); + return json.is_object() && json.contains("execrc_30") && json.contains("execrc_60") && + json["execrc_30"].is_number_unsigned() && json["execrc_60"].is_number_unsigned() && + json["execrc_30"].get() <= UINT32_MAX && json["execrc_60"].get() <= UINT32_MAX; +} + +bool ConfigureServiceTransport(CURL* curl, const std::string& url, std::string& error) +{ + error.clear(); + const bool websocket = url.compare(0, 6, "wss://") == 0; + if (!curl || (!websocket && url.compare(0, 8, "https://") != 0)) + { + error = "Service connections require HTTPS or WSS."; + return false; + } + auto set = [&](CURLoption option, auto value) { + const CURLcode code = curl_easy_setopt(curl, option, value); + if (code != CURLE_OK && error.empty()) + error = "Unable to configure service transport: " + std::string(curl_easy_strerror(code)); + return code == CURLE_OK; + }; +#if _DEBUG + // debug builds skip certificate checks for local service testing + constexpr long verifyPeer = 0L; + constexpr long verifyHost = 0L; +#else + constexpr long verifyPeer = 1L; + constexpr long verifyHost = 2L; +#endif + return set(CURLOPT_SSL_VERIFYPEER, verifyPeer) && set(CURLOPT_SSL_VERIFYHOST, verifyHost) && + set(CURLOPT_SSL_OPTIONS, static_cast(CURLSSLOPT_NATIVE_CA)) && + set(CURLOPT_DISALLOW_USERNAME_IN_URL, 1L) && set(CURLOPT_FOLLOWLOCATION, 0L) && + set(CURLOPT_PROTOCOLS_STR, websocket ? "wss" : "https") && set(CURLOPT_PROXY, ""); +} +} From 680cbf65b47d390764be1f0f0b18cf02da53c0d4 Mon Sep 17 00:00:00 2001 From: tintinhamans <5984296+tintinhamans@users.noreply.github.com> Date: Thu, 22 Oct 2026 23:46:47 +0200 Subject: [PATCH 17/17] fix(online): Log response bodies, login codes and websocket payloads only with verbose logging --- .../GeneralsOnline/HTTP/HTTPRequest.cpp | 57 ++++++++------ .../GeneralsOnline/OnlineServices_Auth.cpp | 27 ++++--- .../OnlineServices_RoomsInterface.cpp | 75 ++++++++++++++++--- 3 files changed, 115 insertions(+), 44 deletions(-) diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp index 2fc21306ccd..dae4491b34f 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/HTTP/HTTPRequest.cpp @@ -51,7 +51,10 @@ void HTTPRequest::SetPostData(const char* szPostData) // TODO_HTTP: Error if verb isnt post m_strPostData = std::string(szPostData); - NetworkLog(ELogVerbosity::LOG_DEBUG, "[%p|%s|Verb %d] Transfer is created: Body is %s", this, m_strURI.c_str(), m_httpVerb, szPostData); + if (NGMP_OnlineServicesManager::Settings.Debug_VerboseLogging()) + { + NetworkLog(ELogVerbosity::LOG_DEBUG, "[%p|%s|Verb %d] Transfer is created: Body is %s", this, m_strURI.c_str(), m_httpVerb, szPostData); + } } void HTTPRequest::SetPostDataBuffer(std::vector vecBuffer) @@ -68,7 +71,10 @@ void HTTPRequest::StartRequest() m_currentBufSize_Used = 0; - NetworkLog(ELogVerbosity::LOG_DEBUG, "[%p|%s|Verb %d] Transfer is starting: Body is %s", this, m_strURI.c_str(), m_httpVerb, m_strPostData.c_str()); + if (NGMP_OnlineServicesManager::Settings.Debug_VerboseLogging()) + { + NetworkLog(ELogVerbosity::LOG_DEBUG, "[%p|%s|Verb %d] Transfer is starting: Body is %s", this, m_strURI.c_str(), m_httpVerb, m_strPostData.c_str()); + } PlatformStartRequest(); } @@ -191,34 +197,37 @@ void HTTPRequest::Threaded_SetComplete(CURLcode result) } #endif - std::string strResponse = std::string(reinterpret_cast(m_vecBuffer.data()), m_currentBufSize_Used); - NetworkLog(ELogVerbosity::LOG_RELEASE, "[%p|%s|Verb %d] Transfer is complete: %d bytes total! Curl result is %d", this, strURIRedacted.c_str(), m_httpVerb, m_currentBufSize_Used, result); - - // if we got an error, set the response code to 0 + // response bodies can carry tokens and codes, so only their size is logged unless verbose + NetworkLog(ELogVerbosity::LOG_RELEASE, "[%p|%s|Verb %d] Completed: curl=%d status=%ld bytes=%zu", this, strURIRedacted.c_str(), m_httpVerb, result, m_responseCode, m_currentBufSize_Used); + if (NGMP_OnlineServicesManager::Settings.Debug_VerboseLogging()) + { + std::string strResponse(reinterpret_cast(m_vecBuffer.data()), m_currentBufSize_Used); #if !_DEBUG - static const std::string strSeedKey = "\"RNGSeed\":"; - for (size_t seedPos = strResponse.find(strSeedKey); seedPos != std::string::npos; seedPos = strResponse.find(strSeedKey, seedPos)) - { - size_t valueStart = seedPos + strSeedKey.length(); - size_t valueEnd = strResponse.find_first_of(",}", valueStart); - if (valueEnd == std::string::npos) - break; + // release builds keep hiding the RNG seed and anything token related even when verbose + static const std::string strSeedKey = "\"RNGSeed\":"; + for (size_t seedPos = strResponse.find(strSeedKey); seedPos != std::string::npos; seedPos = strResponse.find(strSeedKey, seedPos)) + { + size_t valueStart = seedPos + strSeedKey.length(); + size_t valueEnd = strResponse.find_first_of(",}", valueStart); + if (valueEnd == std::string::npos) + break; - strResponse.replace(valueStart, valueEnd - valueStart, ""); - seedPos = valueStart; - } + strResponse.replace(valueStart, valueEnd - valueStart, ""); + seedPos = valueStart; + } - std::string strResponseLower = strResponse; - std::transform(strResponseLower.begin(), strResponseLower.end(), strResponseLower.begin(), - [](unsigned char c) { return std::tolower(c); }); - if (strResponseLower.find("token") != std::string::npos) - { - strResponse = ""; - } + std::string strResponseLower = strResponse; + std::transform(strResponseLower.begin(), strResponseLower.end(), strResponseLower.begin(), + [](unsigned char c) { return std::tolower(c); }); + if (strResponseLower.find("token") != std::string::npos) + { + strResponse = ""; + } #endif - NetworkLog(ELogVerbosity::LOG_RELEASE, "[%p|%s|Verb %d] Response was %d - %s!", this, strURIRedacted.c_str(), m_httpVerb, m_responseCode, strResponse.c_str()); + NetworkLog(ELogVerbosity::LOG_RELEASE, "[%p|%s|Verb %d] Response was %ld - %s!", this, strURIRedacted.c_str(), m_httpVerb, m_responseCode, strResponse.c_str()); + } // trigger callback InvokeCallbackIfComplete(); diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Auth.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Auth.cpp index 40f85720725..f33fcd93bae 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Auth.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_Auth.cpp @@ -173,12 +173,15 @@ void NGMP_OnlineServices_AuthInterface::SendMiddlewareToken(std::string strMWTok void NGMP_OnlineServices_AuthInterface::OnRefreshTokenFailed(const char* szReason, const std::string& strBody) { - // the body can hold a rotated refresh token, so release builds log only its size -#if _DEBUG - NetworkLog(ELogVerbosity::LOG_RELEASE, "[AUTH]: Refresh response body: %s", strBody.c_str()); -#else - NetworkLog(ELogVerbosity::LOG_RELEASE, "[AUTH]: Refresh response body was %zu bytes", strBody.size()); -#endif + // the body can hold a rotated refresh token, so only its size is logged unless verbose + if (NGMP_OnlineServicesManager::Settings.Debug_VerboseLogging()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[AUTH]: Refresh response body: %s", strBody.c_str()); + } + else + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[AUTH]: Refresh response body was %zu bytes", strBody.size()); + } if (m_currentRefreshAttempt < m_maxRefreshAttempts) { @@ -457,7 +460,10 @@ void NGMP_OnlineServices_AuthInterface::DoFullLoginFlow() m_lastCheckCode = std::chrono::duration_cast(std::chrono::utc_clock::now().time_since_epoch()).count(); m_strCode = authResp.login_code; - NetworkLog(ELogVerbosity::LOG_DEBUG, "Login Code is %s", m_strCode.c_str()); + if (NGMP_OnlineServicesManager::Settings.Debug_VerboseLogging()) + { + NetworkLog(ELogVerbosity::LOG_DEBUG, "Login Code is %s", m_strCode.c_str()); + } #if defined(USE_TEST_ENV) std::string strURI = std::format("https://www.playgenerals.online/login/?gamecode={}&env=test", m_strCode.c_str()); @@ -562,9 +568,10 @@ void NGMP_OnlineServices_AuthInterface::Tick() nlohmann::json jsonObject = nlohmann::json::parse(strBody); AuthResponse authResp = jsonObject.get(); -#if _DEBUG - NetworkLog(ELogVerbosity::LOG_RELEASE, "PageBody: %s", strBody.c_str()); -#endif + if (NGMP_OnlineServicesManager::Settings.Debug_VerboseLogging()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "PageBody: %s", strBody.c_str()); + } if (authResp.result == EAuthResponseResult::CODE_INVALID) { NetworkLog(ELogVerbosity::LOG_RELEASE, "LOGIN: Code didnt exist, trying again soon"); diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp index b91f9c9df63..928175632fd 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/OnlineServices_RoomsInterface.cpp @@ -617,12 +617,26 @@ static bool JSONDeserialize(const char* szBuffer, nlohmann::json* jsonObject) } catch (nlohmann::json::exception& jsonException) { - NetworkLog(ELogVerbosity::LOG_RELEASE, "JSONDeserialize: Unparsable JSON: %s (%s)", szBuffer, jsonException.what()); + if (NGMP_OnlineServicesManager::Settings.Debug_VerboseLogging()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "JSONDeserialize: Unparsable JSON: %s (%s)", szBuffer, jsonException.what()); + } + else + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "JSONDeserialize: Invalid message JSON"); + } return false; } catch (...) { - NetworkLog(ELogVerbosity::LOG_RELEASE, "JSONDeserialize: Unparsable JSON: %s", szBuffer); + if (NGMP_OnlineServicesManager::Settings.Debug_VerboseLogging()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "JSONDeserialize: Unparsable JSON: %s", szBuffer); + } + else + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "JSONDeserialize: Invalid message JSON"); + } return false; } @@ -641,7 +655,14 @@ static bool JSONGetAsObject(nlohmann::json& jsonObject, T* outMsg) catch (nlohmann::json::exception& jsonException) { std::string targetTypeName = typeid(T).name(); - NetworkLog(ELogVerbosity::LOG_RELEASE, "JSONGetAsObject: Unparsable JSON: Target Type is %s (%s)", targetTypeName.c_str(), jsonException.what()); + if (NGMP_OnlineServicesManager::Settings.Debug_VerboseLogging()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "JSONGetAsObject: Unparsable JSON: Target Type is %s (%s)", targetTypeName.c_str(), jsonException.what()); + } + else + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "JSONGetAsObject: Invalid message type %s", targetTypeName.c_str()); + } return false; } catch (...) @@ -1012,7 +1033,14 @@ CURLcode WebSocket::ReceiveOneFrame() if (ret == CURLE_OK) { - NetworkLog(ELogVerbosity::LOG_DEBUG, "Got websocket msg: %s", bufferThisRecv); + if (NGMP_OnlineServicesManager::Settings.Debug_VerboseLogging()) + { + NetworkLog(ELogVerbosity::LOG_DEBUG, "Got websocket msg: %s", bufferThisRecv); + } + else + { + NetworkLog(ELogVerbosity::LOG_DEBUG, "[WebSocket] Received %zu bytes", rlen); + } NetworkLog(ELogVerbosity::LOG_DEBUG, "Got websocket len: %d", rlen); // any server frame proves liveness, not just a JSON PONG @@ -1645,7 +1673,14 @@ CURLcode WebSocket::ReceiveOneFrame() if (bParsed) { - NetworkLog(ELogVerbosity::LOG_RELEASE, "[PROBE] GOT PROBE REQUEST: %s!", probe.url.c_str()); + if (NGMP_OnlineServicesManager::Settings.Debug_VerboseLogging()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[PROBE] GOT PROBE REQUEST: %s!", probe.url.c_str()); + } + else + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[PROBE] Received request"); + } NGMP_OnlineServicesManager::GetInstance()->CaptureScreenshotForProbe(EScreenshotType::SCREENSHOT_TYPE_GAMEPLAY, probe.url); @@ -1834,21 +1869,41 @@ CURLcode WebSocket::ReceiveOneFrame() } catch (nlohmann::json::exception& jsonException) { - - NetworkLog(ELogVerbosity::LOG_RELEASE, "Unparsable WebSocketMessage 101: %s (JSON: %s)", bufferThisRecv, jsonException.what()); - NetworkLog(ELogVerbosity::LOG_RELEASE, "Buildup buffer is: %s", m_vecWSPartialBuffer.data()); + if (NGMP_OnlineServicesManager::Settings.Debug_VerboseLogging()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Unparsable WebSocketMessage 101: %s (JSON: %s)", bufferThisRecv, jsonException.what()); + NetworkLog(ELogVerbosity::LOG_RELEASE, "Buildup buffer is: %s", m_vecWSPartialBuffer.data()); + } + else + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[WebSocket] Invalid JSON message"); + } m_vecWSPartialBuffer.clear(); } catch (std::exception& e) { - NetworkLog(ELogVerbosity::LOG_RELEASE, "Unparsable WebSocketMessage 100: %s (%s)", bufferThisRecv, e.what()); + if (NGMP_OnlineServicesManager::Settings.Debug_VerboseLogging()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Unparsable WebSocketMessage 100: %s (%s)", bufferThisRecv, e.what()); + } + else + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[WebSocket] Message processing failed"); + } m_vecWSPartialBuffer.clear(); } catch (...) { - NetworkLog(ELogVerbosity::LOG_RELEASE, "Unparsable WebSocketMessage 102: %s", bufferThisRecv); + if (NGMP_OnlineServicesManager::Settings.Debug_VerboseLogging()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Unparsable WebSocketMessage 102: %s", bufferThisRecv); + } + else + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "[WebSocket] Message processing failed"); + } m_vecWSPartialBuffer.clear(); }