diff --git a/Core/GameEngine/CMakeLists.txt b/Core/GameEngine/CMakeLists.txt index e6aee6da85f..8f5892206e0 100644 --- a/Core/GameEngine/CMakeLists.txt +++ b/Core/GameEngine/CMakeLists.txt @@ -15,6 +15,7 @@ set(GAMEENGINE_SRC # Include/Common/BattleHonors.h # Include/Common/BezFwdIterator.h # Include/Common/BezierSegment.h + Include/Common/BIGFileSystem.h # Include/Common/BitFlags.h # Include/Common/BitFlagsIO.h # Include/Common/BorderColors.h @@ -644,6 +645,7 @@ set(GAMEENGINE_SRC Source/Common/System/ArchiveFile.cpp Source/Common/System/ArchiveFileSystem.cpp Source/Common/System/AsciiString.cpp + Source/Common/System/BIGFileSystem.cpp # Source/Common/System/BuildAssistant.cpp # Source/Common/System/CriticalSection.cpp # Source/Common/System/DataChunk.cpp diff --git a/Core/GameEngine/Include/Common/ArchiveFile.h b/Core/GameEngine/Include/Common/ArchiveFile.h index 63987f9c9cb..61a43117017 100644 --- a/Core/GameEngine/Include/Common/ArchiveFile.h +++ b/Core/GameEngine/Include/Common/ArchiveFile.h @@ -47,26 +47,35 @@ class File; class ArchiveFile { public: - ArchiveFile(); - virtual ~ArchiveFile(); + ArchiveFile(AsciiString name, AsciiString path); + ~ArchiveFile(); - virtual Bool getFileInfo( const AsciiString& filename, FileInfo *fileInfo) const = 0; ///< fill in the fileInfo struct with info about the file requested. - virtual File* openFile( const Char *filename, Int access = 0) = 0; ///< Open the specified file within the archive file - virtual void closeAllFiles() = 0; ///< Close all file opened in this archive file - virtual AsciiString getName() = 0; ///< Returns the name of the archive file - virtual AsciiString getPath() = 0; ///< Returns full path and name of archive file - virtual void setSearchPriority( Int new_priority ) = 0; ///< Set this archive file's search priority - virtual void close() = 0; ///< Close this archive file + Bool getFileInfo( const AsciiString& filename, FileInfo *fileInfo) const; ///< fill in the fileInfo struct with info about the file requested. + File* openFile( const Char *filename, Int access = 0); ///< Open the specified file within the archive file + void closeAllFiles(); ///< Close all file opened in this archive file + AsciiString getName(); ///< Returns the name of the archive file + AsciiString getPath(); ///< Returns full path and name of archive file + void setSearchPriority( Int new_priority ); ///< Set this archive file's search priority + void close(); ///< Close this archive file void attachFile(File *file); + void setPriority(Int priority) { m_priority = priority; } + Int getPriority() const { return m_priority; } void getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; - void getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; void addFile(const AsciiString& path, const ArchivedFileInfo *fileInfo); ///< add this file to our directory tree. protected: + void getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; const ArchivedFileInfo * getArchivedFileInfo(const AsciiString& filename) const; ///< return the ArchivedFileInfo from the directory tree. +protected: + Int m_priority = ARCHIVE_PRIORITY_RETAIL; ///< lookup priority in the archive file system, lower wins File *m_file; ///< file pointer to the archive file on disk. Kept open so we don't have to continuously open and close the file all the time. + FileInfo m_fileInfo; ///< file info of the archive file on disk. + AsciiString m_name; ///< archive file name + AsciiString m_path; ///< archive file path + +private: DetailedArchivedDirectoryInfo m_rootDirectory; }; diff --git a/Core/GameEngine/Include/Common/ArchiveFileSystem.h b/Core/GameEngine/Include/Common/ArchiveFileSystem.h index 48d144484c6..381765359ac 100644 --- a/Core/GameEngine/Include/Common/ArchiveFileSystem.h +++ b/Core/GameEngine/Include/Common/ArchiveFileSystem.h @@ -53,6 +53,8 @@ #include "Common/AsciiString.h" #include "Common/FileSystem.h" // for typedefs, etc. #include "Common/STLTypedefs.h" +#include +#include //---------------------------------------------------------------------------- // Forward References @@ -120,6 +122,31 @@ class ArchivedFileInfo }; +// Lookup priority of an archive, lower wins. The archive mounted first wins on equal priority. +enum ArchivePriority +{ + ARCHIVE_PRIORITY_EXCLUDED = -1, ///< not mounted: a BIG of a game folder that matches user.exclude_paths + ARCHIVE_PRIORITY_MOD = 0, ///< -mod and -modDir, 0 to 999, a later one is lower and wins + ARCHIVE_PRIORITY_GAME_FOLDER = 1000, ///< BIGs of the Zero Hour folder that are not retail, in name order + ARCHIVE_PRIORITY_GAME_FOLDER_GENERALS = 1100, ///< BIGs of the Generals data (ZH_Generals or the Generals install) that are not retail + ARCHIVE_PRIORITY_USER = 100000, ///< data_packs.user of settings.json, 100000 plus the position in its include_paths + ARCHIVE_PRIORITY_GENERALS_ONLINE = 1000000, ///< data_packs.generals_online, 1000000 plus the position in its include_paths + ARCHIVE_PRIORITY_COMMUNITY_PATCH = 2000000, ///< data_packs.community_patch, 2000000 plus the position in its include_paths + ARCHIVE_PRIORITY_RETAIL = 3000000, ///< Zero Hour retail and official patch BIGs + ARCHIVE_PRIORITY_RETAIL_GENERALS = 3000100 ///< Generals retail and official patch BIGs +}; + +// One entry of the include_paths of a data pack layer in settings.json, see ArchiveFileSystem::parseMountEntry +struct ArchiveMountEntry +{ + enum Kind { KIND_BIG, KIND_BIG_GLOB }; + enum Base { BASE_GAME, BASE_USER, BASE_ABSOLUTE }; + + Kind m_kind; + Base m_base; + AsciiString m_path; ///< with backslashes, relative to the base, the BASE_ABSOLUTE path is complete +}; + class ArchiveFileSystem : public SubsystemInterface { public: @@ -141,12 +168,22 @@ class ArchiveFileSystem : public SubsystemInterface void getFileListInDirectory(const AsciiString& currentDirectory, const AsciiString& originalDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const; ///< search the given directory for files matching the searchName (egs. *.ini, *.rep). Possibly search subdirectories. Scans each Archive file. Bool getFileInfo(const AsciiString& filename, FileInfo *fileInfo, FileInstance instance = 0) const; ///< see FileSystem.h - virtual Bool loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) = 0; + virtual Bool loadArchiveFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) = 0; - // Unprotected this for copy-protection routines - ArchiveFile* getArchiveFile(const AsciiString& filename, FileInstance instance = 0) const; + virtual void loadMods() = 0; + + /// Parses an entry of the include_paths of a data pack layer: "game:", "user:" or an absolute path. A path ending in .big is one BIG, + /// a last component with * or ? and ending in .big is a set of BIGs, a path ending in a slash is every BIG in that folder. + /// Returns FALSE and a reason for an entry that is not valid. + static Bool parseMountEntry(const AsciiString& text, ArchiveMountEntry* entry, AsciiString* reason); + + /// TRUE for the retail archives and the official patches of Zero Hour and Generals, in every language. + static Bool isGeneralsArchivePath(const AsciiString& path, Bool generalsFolder); ///< TRUE for the Generals archives: the Generals install folder, or a ZH_Generals folder + + static Bool isRetailArchive(const AsciiString& path, Bool generalsFolder, Bool zeroHour); - void loadMods(); + /// BIGs of the game folders that are not retail and not excluded by user.exclude_paths, which are mounted above the data pack layers + const std::vector& getGameFolderArchives() const { return m_gameFolderArchives; } ArchivedDirectoryInfo* friend_getArchivedDirectoryInfo(const Char* directory); @@ -160,11 +197,21 @@ class ArchiveFileSystem : public SubsystemInterface AsciiString lastToken; ///< Synonymous for file name if the search directory was a file path }; + ArchiveFile* getArchiveFile(const AsciiString& filename, FileInstance instance = 0) const; ArchivedDirectoryInfoResult getArchivedDirectoryInfo(const Char* directory); - virtual void loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool overwrite = FALSE, Bool sortedByName = FALSE); ///< load the archive file's header information and apply it to the global archive directory tree. sortedByName inserts by archive filename order (addon number convention), ignoring overwrite. + virtual void loadIntoDirectoryTree(ArchiveFile *archiveFile, Int priority = ARCHIVE_PRIORITY_RETAIL); ///< load the archive file's header information and apply it to the global archive directory tree. The lowest priority number wins, the first mounted wins on equal priority. + + Int getNextModPriority() { return m_nextModPriority > ARCHIVE_PRIORITY_MOD ? m_nextModPriority-- : ARCHIVE_PRIORITY_MOD; } ///< -mod and -modDir: a later one wins + Int getGameFolderArchivePriority(const AsciiString& path, Bool generalsFolder, Bool zeroHour); ///< retail archives are the bottom, any other BIG of the game folders ranks above the data pack layers unless user.exclude_paths matches it + + void mountListedArchive(const ArchiveMountEntry& entry, Int priority, const PathExcludeList* excludes); + void mountDataPackLayer(const char* layer, Bool enabled, const std::vector& includePaths, const PathExcludeList* excludes, Int priority); ///< mounts the include_paths of one layer in list order, the first one wins, minus what excludes matches ArchiveFileMap m_archiveFileMap; + std::vector m_gameFolderArchives; + AsciiString m_generalsBaseSource; ///< where the Generals data came from, for the log + Int m_nextModPriority = ARCHIVE_PRIORITY_MOD + 999; ArchivedDirectoryInfo m_rootDirectory; }; diff --git a/Core/GameEngineDevice/Include/Win32Device/Common/Win32BIGFileSystem.h b/Core/GameEngine/Include/Common/BIGFileSystem.h similarity index 85% rename from Core/GameEngineDevice/Include/Win32Device/Common/Win32BIGFileSystem.h rename to Core/GameEngine/Include/Common/BIGFileSystem.h index 6819ba90577..95e20c0ea6c 100644 --- a/Core/GameEngineDevice/Include/Win32Device/Common/Win32BIGFileSystem.h +++ b/Core/GameEngine/Include/Common/BIGFileSystem.h @@ -22,7 +22,7 @@ // // //////////////////////////////////////////////////////////////////////////////// -//////// Win32BIGFileSystem.h /////////////////////////// +//////// BIGFileSystem.h /////////////////////////// // Bryan Cleveland, August 2002 ///////////////////////////////////////////////////////////// @@ -30,11 +30,11 @@ #include "Common/ArchiveFileSystem.h" -class Win32BIGFileSystem : public ArchiveFileSystem +class BIGFileSystem : public ArchiveFileSystem { public: - Win32BIGFileSystem(); - virtual ~Win32BIGFileSystem() override; + BIGFileSystem(); + virtual ~BIGFileSystem() override; virtual void init() override; virtual void update() override; @@ -49,7 +49,9 @@ class Win32BIGFileSystem : public ArchiveFileSystem virtual void closeArchiveFile(const Char *filename) override; virtual void closeAllFiles() override; ///< Close all files associated with ArchiveFiles - virtual Bool loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) override; + virtual Bool loadArchiveFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) override; + + virtual void loadMods() override; protected: }; diff --git a/Core/GameEngine/Include/Common/FileSystem.h b/Core/GameEngine/Include/Common/FileSystem.h index 8a8b99a9f8c..d4a518434d6 100644 --- a/Core/GameEngine/Include/Common/FileSystem.h +++ b/Core/GameEngine/Include/Common/FileSystem.h @@ -53,6 +53,9 @@ #include +#include +#include + #include "WWLib/mutex.h" //---------------------------------------------------------------------------- @@ -109,6 +112,14 @@ typedef UnsignedByte FileInstance; struct FileInfo { + FileInfo() + : sizeHigh(0) + , sizeLow(0) + , timestampHigh(0) + , timestampLow(0) + { + } + Int64 size() const { return (Int64)sizeHigh << 32 | sizeLow; } Int64 timestamp() const { return (Int64)timestampHigh << 32 | timestampLow; } @@ -118,6 +129,30 @@ struct FileInfo { Int timestampLow; }; +//=============================== +// PathExcludeList +//=============================== +/** + * The exclude_paths of settings.json. An entry is game:, user: or an absolute pattern, where * matches any text + * including folders and ? one character. A pattern ending in a slash is a folder and everything in it. Matching ignores case and slash direction. + * A path is tested as a key: game:, user: or the absolute path. + */ +class PathExcludeList +{ +public: + static Bool parse(const std::string& text, std::string* pattern, AsciiString* reason); ///< FALSE and a reason for an entry that is not valid + static std::string makeKey(const Char* path); ///< key of a path that is relative to the game folder or absolute + static std::string makeKey(const Char* path, const Char* userDataPath); ///< same, a path inside the user data folder becomes user: + + void set(const std::vector& entries); ///< entries that are not valid are dropped + Bool isEmpty() const { return m_patterns.empty(); } + Bool matches(const std::string& key) const; + Bool matchesPath(const Char* path) const { return !m_patterns.empty() && matches(makeKey(path)); } + +private: + std::vector m_patterns; +}; + //=============================== // FileSystem //=============================== @@ -157,6 +192,10 @@ class FileSystem : public SubsystemInterface Bool createDirectory(AsciiString directory); ///< create a directory of the given name. + /// Loose files of the game folder that match are treated as if they did not exist: not opened, not found, not listed. Files that are written and files outside the game folder are not affected. + void setExcludedPaths(const std::vector& entries) { m_excludedPaths.set(entries); } + const PathExcludeList& getExcludedPaths() const { return m_excludedPaths; } + static AsciiString normalizePath(const AsciiString& path); ///< normalizes a file path. The path can refer to a directory. File path must be absolute, but does not need to exist. Returns an empty string on failure. static Bool isPathInDirectory(const AsciiString& testPath, const AsciiString& basePath); ///< determines if a file path is within a base path. Both paths must be absolute, but do not need to exist. @@ -164,6 +203,10 @@ class FileSystem : public SubsystemInterface static bool removeExtension(UnicodeString& path); protected: + Bool isLooseFileExcluded(const Char *filename) const { return m_excludedPaths.matchesPath(filename); } + + PathExcludeList m_excludedPaths; + #if ENABLE_FILESYSTEM_EXISTENCE_CACHE struct FileExistData { diff --git a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp index 1f982b65a1d..9e0db3f0ee2 100644 --- a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp +++ b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp @@ -30,8 +30,12 @@ #include "Common/ArchiveFile.h" #include "Common/ArchiveFileSystem.h" +#include "Common/LocalFileSystem.h" #include "Common/file.h" +#include "Common/LocalFile.h" #include "Common/PerfTimer.h" +#include "Common/RAMFile.h" +#include "Common/StreamingArchiveFile.h" // checks to see if str matches searchString. Search string is done in the @@ -89,8 +93,10 @@ ArchiveFile::~ArchiveFile() } } -ArchiveFile::ArchiveFile() +ArchiveFile::ArchiveFile(AsciiString name, AsciiString path) : m_file(nullptr) + , m_name(name) + , m_path(path) { } @@ -106,13 +112,11 @@ void ArchiveFile::addFile(const AsciiString& path, const ArchivedFileInfo *fileI while (!token.isEmpty()) { DetailedArchivedDirectoryInfoMap::iterator tempiter = dirInfo->m_directories.find(token); - if (tempiter == dirInfo->m_directories.end()) - { + if (tempiter == dirInfo->m_directories.end()) { dirInfo = &(dirInfo->m_directories[token]); dirInfo->m_directoryName = token; } - else - { + else { dirInfo = &tempiter->second; } @@ -132,18 +136,12 @@ void ArchiveFile::getFileListInDirectory(const AsciiString& currentDirectory, co tokenizer.nextToken(&token, "\\/"); while (!token.isEmpty()) { - DetailedArchivedDirectoryInfoMap::const_iterator it = dirInfo->m_directories.find(token); - if (it != dirInfo->m_directories.end()) - { - dirInfo = &it->second; - } - else - { - // if the directory doesn't exist, then there aren't any files to be had. + // if the directory doesn't exist, then there aren't any files to be had. + if (it == dirInfo->m_directories.end()) return; - } + dirInfo = &it->second; tokenizer.nextToken(&token, "\\/"); } @@ -153,33 +151,31 @@ void ArchiveFile::getFileListInDirectory(const AsciiString& currentDirectory, co void ArchiveFile::getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const { DetailedArchivedDirectoryInfoMap::const_iterator diriter = dirInfo->m_directories.begin(); - while (diriter != dirInfo->m_directories.end()) { + for (; diriter != dirInfo->m_directories.end(); ++diriter) { const DetailedArchivedDirectoryInfo *tempDirInfo = &(diriter->second); - AsciiString tempdirname; - tempdirname = currentDirectory; - if ((!tempdirname.isEmpty()) && (!tempdirname.endsWith("\\"))) { - tempdirname.concat('\\'); + AsciiString tempDirName = currentDirectory; + if (!tempDirName.isEmpty() && !tempDirName.endsWith("\\")) { + tempDirName.concat('\\'); } - tempdirname.concat(tempDirInfo->m_directoryName); - getFileListInDirectory(tempDirInfo, tempdirname, searchName, filenameList, searchSubdirectories); - diriter++; + tempDirName.concat(tempDirInfo->m_directoryName); + getFileListInDirectory(tempDirInfo, tempDirName, searchName, filenameList, searchSubdirectories); } ArchivedFileInfoMap::const_iterator fileiter = dirInfo->m_files.begin(); - while (fileiter != dirInfo->m_files.end()) { - if (SearchStringMatches(fileiter->second.m_filename, searchName)) { + for (; fileiter != dirInfo->m_files.end(); ++fileiter) { + const ArchivedFileInfo &fileInfo = fileiter->second; + if (SearchStringMatches(fileInfo.m_filename, searchName)) { AsciiString tempfilename; tempfilename = currentDirectory; - if ((!tempfilename.isEmpty()) && (!tempfilename.endsWith("\\"))) { + if (!tempfilename.isEmpty() && !tempfilename.endsWith("\\")) { tempfilename.concat('\\'); } - tempfilename.concat(fileiter->second.m_filename); + tempfilename.concat(fileInfo.m_filename); if (filenameList.find(tempfilename) == filenameList.end()) { // only insert into the list if its not already in there. filenameList.insert(tempfilename); } } - fileiter++; } } @@ -189,7 +185,18 @@ void ArchiveFile::attachFile(File *file) m_file->close(); m_file = nullptr; } + m_file = file; + + if (m_file != nullptr) { + if (!TheLocalFileSystem->getFileInfo(m_file->getName(), &m_fileInfo)) { + DEBUG_CRASH(("ArchiveFile::attachFile - Cannot query file info for '%s' unexpectedly", m_file->getName())); + m_fileInfo = FileInfo(); + } + } + else { + m_fileInfo = FileInfo(); + } } const ArchivedFileInfo * ArchiveFile::getArchivedFileInfo(const AsciiString& filename) const @@ -201,7 +208,7 @@ const ArchivedFileInfo * ArchiveFile::getArchivedFileInfo(const AsciiString& fil tokenizer.toLower(); tokenizer.nextToken(&token, "\\/"); - while (!token.find('.') || tokenizer.find('.')) + while (!tokenizer.isEmpty()) { DetailedArchivedDirectoryInfoMap::const_iterator it = dirInfo->m_directories.find(token); if (it != dirInfo->m_directories.end()) @@ -227,3 +234,116 @@ const ArchivedFileInfo * ArchiveFile::getArchivedFileInfo(const AsciiString& fil } } + +//============================================================================ +// ArchiveFile::openFile +//============================================================================ + +File* ArchiveFile::openFile( const Char *filename, Int access ) +{ + const ArchivedFileInfo *fileInfo = getArchivedFileInfo(AsciiString(filename)); + + if (fileInfo == nullptr) { + return nullptr; + } + + RAMFile *ramFile = nullptr; + + if (BitIsSet(access, File::STREAMING)) + ramFile = newInstance( StreamingArchiveFile ); + else + ramFile = newInstance( RAMFile ); + + ramFile->deleteOnClose(); + if (ramFile->openFromArchive(m_file, fileInfo->m_filename, fileInfo->m_offset, fileInfo->m_size) == FALSE) { + ramFile->close(); + ramFile = nullptr; + return nullptr; + } + + if ((access & File::WRITE) == 0) { + // requesting read only access. Just return the RAM file. + return ramFile; + } + + // whoever is opening this file wants write access, so copy the file to the local disk + // and return that file pointer. + + constexpr size_t bufferSize = 0; + File *localFile = TheLocalFileSystem->openFile(filename, access, bufferSize); + if (localFile != nullptr) { + ramFile->copyDataToFile(localFile); + } + + ramFile->close(); + ramFile = nullptr; + + return localFile; +} + +//============================================================================ +// ArchiveFile::closeAllFiles +//============================================================================ + +void ArchiveFile::closeAllFiles() +{ + +} + +//============================================================================ +// ArchiveFile::getName +//============================================================================ + +AsciiString ArchiveFile::getName() +{ + return m_name; +} + +//============================================================================ +// ArchiveFile::getPath +//============================================================================ + +AsciiString ArchiveFile::getPath() +{ + return m_path; +} + +//============================================================================ +// ArchiveFile::setSearchPriority +//============================================================================ + +void ArchiveFile::setSearchPriority( Int new_priority ) +{ + +} + +//============================================================================ +// ArchiveFile::close +//============================================================================ + +void ArchiveFile::close() +{ + +} + +//============================================================================ +// ArchiveFile::getFileInfo +//============================================================================ + +Bool ArchiveFile::getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const +{ + const ArchivedFileInfo *archivedFileInfo = getArchivedFileInfo(filename); + + if (archivedFileInfo == nullptr) { + return FALSE; + } + + // fill in the size info. Since the size can't be bigger than a JUNK file, the high int will always be 0. + fileInfo->sizeHigh = 0; + fileInfo->sizeLow = archivedFileInfo->m_size; + // the timestamp is taken from the archive file. + fileInfo->timestampHigh = m_fileInfo.timestampHigh; + fileInfo->timestampLow = m_fileInfo.timestampLow; + + return TRUE; +} diff --git a/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp b/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp index 5720c5b9417..6171f26fabe 100644 --- a/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp +++ b/Core/GameEngine/Source/Common/System/ArchiveFileSystem.cpp @@ -46,11 +46,16 @@ //---------------------------------------------------------------------------- #include "PreRTS.h" +#include +#include +#include +#include #include "Common/ArchiveFile.h" #include "Common/ArchiveFileSystem.h" #include "Common/AsciiString.h" #include "Common/LocalFileSystem.h" #include "Common/PerfTimer.h" +#include "Common/Registry.h" #include "../NGMP_include.h" #include "../OnlineServices_Init.h" @@ -127,8 +132,9 @@ ArchiveFileSystem::~ArchiveFileSystem() } } -void ArchiveFileSystem::loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool overwrite, Bool sortedByName) +void ArchiveFileSystem::loadIntoDirectoryTree(ArchiveFile *archiveFile, Int priority) { + archiveFile->setPriority(priority); FilenameList filenameList; @@ -144,9 +150,9 @@ void ArchiveFileSystem::loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool ove AsciiString token; AsciiString tokenizer = *it; tokenizer.toLower(); - Bool infoInPath = tokenizer.nextToken(&token, "\\/"); + tokenizer.nextToken(&token, "\\/"); - while (infoInPath && (!token.find('.') || tokenizer.find('.'))) + while (!tokenizer.isEmpty()) { path.concat(token); path.concat('\\'); @@ -163,119 +169,311 @@ void ArchiveFileSystem::loadIntoDirectoryTree(ArchiveFile *archiveFile, Bool ove dirInfo = &tempiter->second; } - infoInPath = tokenizer.nextToken(&token, "\\/"); + tokenizer.nextToken(&token, "\\/"); } - ArchivedFileLocationMap::iterator fileIt; - if (sortedByName) - { - // Insert by case-insensitive archive filename, matching game folder load order where the alphabetically first archive wins. - const AsciiString baseName = getBaseFilename(archiveFile->getName()); - std::pair range = dirInfo->m_files.equal_range(token); - fileIt = range.first; - while (fileIt != range.second && getBaseFilename(fileIt->second->getName()).compareNoCase(baseName) <= 0) - ++fileIt; - } - else if (overwrite) + // Insert behind every archive of the same or a lower priority number, so the first one mounted wins on equal priority. + // A plain insert goes to the end of the range of equal keys, the range is rebuilt when the new archive belongs in front of others. + std::pair range = dirInfo->m_files.equal_range(token); + std::vector ordered; + for (ArchivedFileLocationMap::iterator rangeIt = range.first; rangeIt != range.second; ++rangeIt) + ordered.push_back(rangeIt->second); + + size_t position = 0; + while (position < ordered.size() && ordered[position]->getPriority() <= priority) + ++position; + + if (position == ordered.size()) { - // When overwriting, try place the new value at the beginning of the key list. - fileIt = dirInfo->m_files.find(token); + dirInfo->m_files.insert(std::make_pair(token, archiveFile)); } else { - // Append to the end of the key list. - fileIt = dirInfo->m_files.end(); + ordered.insert(ordered.begin() + position, archiveFile); + dirInfo->m_files.erase(range.first, range.second); + for (size_t k = 0; k < ordered.size(); ++k) + dirInfo->m_files.insert(std::make_pair(token, ordered[k])); } - dirInfo->m_files.insert(fileIt, std::make_pair(token, archiveFile)); - -#if defined(DEBUG_LOGGING) && ENABLE_FILESYSTEM_LOGGING - { - const stl::const_range range = stl::get_range(dirInfo->m_files, token, 0); - if (range.distance() >= 2) - { - ArchivedFileLocationMap::const_iterator rangeIt0; - ArchivedFileLocationMap::const_iterator rangeIt1; - - if (overwrite) - { - rangeIt0 = range.begin; - rangeIt1 = std::next(rangeIt0); - - DEBUG_LOG(("ArchiveFileSystem::loadIntoDirectoryTree - adding file %s, archived in %s, overwriting same file in %s", - it->str(), - rangeIt0->second->getName().str(), - rangeIt1->second->getName().str() - )); - } - else - { - rangeIt1 = std::prev(range.end); - rangeIt0 = std::prev(rangeIt1); - - DEBUG_LOG(("ArchiveFileSystem::loadIntoDirectoryTree - adding file %s, archived in %s, overwritten by same file in %s", - it->str(), - rangeIt1->second->getName().str(), - rangeIt0->second->getName().str() - )); - } - } - else - { - DEBUG_LOG(("ArchiveFileSystem::loadIntoDirectoryTree - adding file %s, archived in %s", it->str(), archiveFile->getName().str())); - } - } -#endif + DEBUG_LOG(("ArchiveFileSystem::loadIntoDirectoryTree - adding file %s, archived in %s, priority %d", it->str(), archiveFile->getName().str(), priority)); it++; } } -void ArchiveFileSystem::loadMods() +// Retail archive names, without the extension and in lower case. Only the language of the game counts as retail. +static void addRetailNames(std::set& names, Bool zeroHour) +{ + auto add = [&names](const std::string& name) + { + std::string lower = name; + for (char& c : lower) + c = static_cast(tolower(static_cast(c))); + names.insert(lower); + }; + + const std::string language = GetRegistryLanguage().str(); + + if (zeroHour) + { + add(language + "ZH"); + add("Audio" + language + "ZH"); + add("Speech" + language + "ZH"); + add("W3D" + language + "ZH"); + for (const char* name : { "AudioZH", "INIZH", "MapsZH", "Music", "MusicZH", "ShadersZH", "SpeechZH", "TerrainZH", "TexturesZH", "W3DZH", "WindowZH", "gensecZH", + "PatchData", "PatchINI", "PatchWindow", "PatchZH" }) + add(name); + } + else + { + add(language); + add("Audio" + language); + add("Speech" + language); + for (const char* name : { "Audio", "INI", "Music", "Patch", "PatchData", "PatchINI", "PatchWindow", "Speech", "Terrain", "Textures", "W3D", "Window", "gensec", "maps", "shaders" }) + add(name); + } +} + +Bool ArchiveFileSystem::isGeneralsArchivePath(const AsciiString& path, Bool generalsFolder) +{ + if (generalsFolder) + return TRUE; + + AsciiString lowerPath = path; + lowerPath.toLower(); + return strstr(lowerPath.str(), "zh_generals\\") != nullptr || strstr(lowerPath.str(), "zh_generals/") != nullptr; +} + +Bool ArchiveFileSystem::isRetailArchive(const AsciiString& path, Bool generalsFolder, Bool zeroHour) +{ + static std::set zeroHourNames; + static std::set generalsNames; + if (zeroHourNames.empty()) + { + addRetailNames(zeroHourNames, TRUE); + addRetailNames(generalsNames, FALSE); + } + + AsciiString name = getBaseFilename(path); + if (!name.endsWithNoCase(".big")) + return FALSE; + name.truncateBy(4); + std::string lower = name.str(); + for (char& c : lower) + c = static_cast(tolower(static_cast(c))); + + // The Generals archives are in the Generals install folder, or in a ZH_Generals folder inside the game folder + const Bool inGeneralsFolder = isGeneralsArchivePath(path, generalsFolder); + + if (zeroHour && !inGeneralsFolder) + return zeroHourNames.count(lower) != 0; + return generalsNames.count(lower) != 0; +} + +Int ArchiveFileSystem::getGameFolderArchivePriority(const AsciiString& path, Bool generalsFolder, Bool zeroHour) +{ + // The Zero Hour archives rank above the Generals ones in their tier, whatever the order of the scan + const Bool generals = zeroHour && isGeneralsArchivePath(path, generalsFolder); + if (isRetailArchive(path, generalsFolder, zeroHour)) + return generals ? ARCHIVE_PRIORITY_RETAIL_GENERALS : ARCHIVE_PRIORITY_RETAIL; + + if (TheFileSystem != nullptr && TheFileSystem->getExcludedPaths().matchesPath(path.str())) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Archive %s is excluded by user.exclude_paths", path.str()); + return ARCHIVE_PRIORITY_EXCLUDED; + } + + m_gameFolderArchives.push_back(path); + return generals ? ARCHIVE_PRIORITY_GAME_FOLDER_GENERALS : ARCHIVE_PRIORITY_GAME_FOLDER; +} + +Bool ArchiveFileSystem::parseMountEntry(const AsciiString& text, ArchiveMountEntry* entry, AsciiString* reason) +{ + auto fail = [reason](const char* why) + { + if (reason != nullptr) + *reason = why; + return FALSE; + }; + + std::string value = text.str(); + // trim + while (!value.empty() && isspace(static_cast(value.back()))) + value.pop_back(); + size_t start = 0; + while (start < value.size() && isspace(static_cast(value[start]))) + ++start; + value = value.substr(start); + if (value.empty()) + return fail("empty"); + + for (char& c : value) + { + if (c == '/') + c = '\\'; + if (static_cast(c) < 32 || c == '<' || c == '>' || c == '"' || c == '|') + return fail("invalid character"); + } + + ArchiveMountEntry::Base base; + std::string path; + std::string lowerPrefix = value.substr(0, 5); + for (char& c : lowerPrefix) + c = static_cast(tolower(static_cast(c))); + if (lowerPrefix == "game:") + { + base = ArchiveMountEntry::BASE_GAME; + path = value.substr(5); + } + else if (lowerPrefix == "user:") + { + base = ArchiveMountEntry::BASE_USER; + path = value.substr(5); + } + else if (value.size() > 3 && isalpha(static_cast(value[0])) && value[1] == ':' && value[2] == '\\') + { + base = ArchiveMountEntry::BASE_ABSOLUTE; + path = value; + } + else if (value.size() > 2 && value[0] == '\\' && value[1] == '\\') + { + base = ArchiveMountEntry::BASE_ABSOLUTE; + path = value; + } + else + { + return fail("needs the prefix game: or user:, or an absolute path"); + } + + if (base != ArchiveMountEntry::BASE_ABSOLUTE) + { + if (path.empty() || path[0] == '\\' || path.find(':') != std::string::npos) + return fail("invalid path"); + } + + // no parent folders + size_t pos = 0; + while (pos <= path.size()) + { + size_t next = path.find('\\', pos); + if (next == std::string::npos) + next = path.size(); + if (path.substr(pos, next - pos) == "..") + return fail("parent folder"); + pos = next + 1; + } + + const size_t lastSeparator = path.find_last_of('\\'); + const std::string lastComponent = lastSeparator == std::string::npos ? path : path.substr(lastSeparator + 1); + const std::string directory = lastSeparator == std::string::npos ? std::string() : path.substr(0, lastSeparator + 1); + if (directory.find_first_of("*?") != std::string::npos) + return fail("wildcards are only allowed in the file name"); + + std::string lowerLast = lastComponent; + for (char& c : lowerLast) + c = static_cast(tolower(static_cast(c))); + + ArchiveMountEntry::Kind kind; + if (lastComponent.empty()) + { + // a folder: every BIG in it + if (directory.empty() || directory == "\\\\") + return fail("invalid path"); + kind = ArchiveMountEntry::KIND_BIG_GLOB; + path += "*.big"; + } + else if (lowerLast.size() < 5 || lowerLast.compare(lowerLast.size() - 4, 4, ".big") != 0) + { + return fail("not a BIG file, a BIG folder or a *.big pattern"); + } + else if (lastComponent.find_first_of("*?") != std::string::npos) + { + kind = ArchiveMountEntry::KIND_BIG_GLOB; + } + else + { + kind = ArchiveMountEntry::KIND_BIG; + } + + if (entry != nullptr) + { + entry->m_kind = kind; + entry->m_base = base; + entry->m_path = path.c_str(); + } + return TRUE; +} + +void ArchiveFileSystem::mountListedArchive(const ArchiveMountEntry& entry, Int priority, const PathExcludeList* excludes) { -#if defined(GENERALS_ONLINE) && defined(GENERALS_ONLINE_COMMUNITY_PATCH_CHANGES) - // load community data patch BIG - if (NGMP_OnlineServicesManager::Settings.DataPacks_UseCommunityPatch() - && !TheGlobalData->m_commandLineData.isCommunityDataPatchDisabled()) - { - std::string strBigPath = std::format("{}GeneralsOnlineGameData/500_900_CommunityPatch_CoreINI.big", TheGlobalData->getPath_UserData().str()); - bool bLoaded = false; - if (TheLocalFileSystem->doesFileExist(strBigPath.c_str())) - { - ArchiveFile* archiveFile = openArchiveFile(strBigPath.c_str()); - if (archiveFile != nullptr) - { - // Sorted by filename so the patch respects the addon number order of BIGs in the game folder. - loadIntoDirectoryTree(archiveFile, FALSE, TRUE); - m_archiveFileMap[AsciiString(strBigPath.c_str())] = archiveFile; - bLoaded = true; - } - } - NetworkLog(ELogVerbosity::LOG_RELEASE, "Loaded community patch (%s): %d", strBigPath.c_str(), bLoaded); - } -#endif - - if (TheGlobalData->m_modBIG.isNotEmpty()) + std::string fullPath = entry.m_path.str(); + if (entry.m_base == ArchiveMountEntry::BASE_USER) + fullPath = std::string(TheGlobalData->getPath_UserData().str()) + fullPath; + + // the files to mount, in name order + std::vector bigPaths; + if (entry.m_kind == ArchiveMountEntry::KIND_BIG) { - ArchiveFile *archiveFile = openArchiveFile(TheGlobalData->m_modBIG.str()); + bigPaths.push_back(fullPath); + } + else + { + const size_t lastSeparator = fullPath.find_last_of('\\'); + const std::string directory = lastSeparator == std::string::npos ? std::string() : fullPath.substr(0, lastSeparator + 1); + const std::string mask = lastSeparator == std::string::npos ? fullPath : fullPath.substr(lastSeparator + 1); + FilenameList files; + TheLocalFileSystem->getFileListInDirectory(AsciiString(directory.c_str()), "", AsciiString(mask.c_str()), files, FALSE); + for (FilenameListIter it = files.begin(); it != files.end(); ++it) + bigPaths.push_back(it->str()); + } - if (archiveFile != nullptr) { - DEBUG_LOG(("ArchiveFileSystem::loadMods - loading %s into the directory tree.", TheGlobalData->m_modBIG.str())); - loadIntoDirectoryTree(archiveFile, TRUE); - m_archiveFileMap[TheGlobalData->m_modBIG] = archiveFile; - DEBUG_LOG(("ArchiveFileSystem::loadMods - %s inserted into the archive file map.", TheGlobalData->m_modBIG.str())); + Int mounted = 0; + for (const std::string& bigPath : bigPaths) + { + if (!TheLocalFileSystem->doesFileExist(bigPath.c_str())) + continue; + if (excludes != nullptr && !excludes->isEmpty() && excludes->matches(PathExcludeList::makeKey(bigPath.c_str(), TheGlobalData->getPath_UserData().str()))) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Archive %s is excluded", bigPath.c_str()); + continue; } - else + if (m_archiveFileMap.find(AsciiString(bigPath.c_str())) != m_archiveFileMap.end()) { - DEBUG_LOG(("ArchiveFileSystem::loadMods - could not openArchiveFile(%s)", TheGlobalData->m_modBIG.str())); + NetworkLog(ELogVerbosity::LOG_RELEASE, "Archive %s is already mounted", bigPath.c_str()); + continue; } + ArchiveFile* archiveFile = openArchiveFile(bigPath.c_str()); + if (archiveFile == nullptr) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Archive %s could not be opened", bigPath.c_str()); + continue; + } + loadIntoDirectoryTree(archiveFile, priority); + m_archiveFileMap[AsciiString(bigPath.c_str())] = archiveFile; + ++mounted; + } + NetworkLog(ELogVerbosity::LOG_RELEASE, "Archive %s at priority %d: %d archives", fullPath.c_str(), priority, mounted); +} + +void ArchiveFileSystem::mountDataPackLayer(const char* layer, Bool enabled, const std::vector& includePaths, const PathExcludeList* excludes, Int priority) +{ + if (!enabled) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Data pack layer %s is disabled", layer); + return; } - if (TheGlobalData->m_modDir.isNotEmpty()) + NetworkLog(ELogVerbosity::LOG_RELEASE, "Data pack layer %s: %d include_paths", layer, static_cast(includePaths.size())); + for (size_t i = 0; i < includePaths.size(); ++i) { - MAYBE_UNUSED Bool ret = loadBigFilesFromDirectory(TheGlobalData->m_modDir, "*.big", TRUE); - (void)ret; - DEBUG_ASSERTLOG(ret, ("loadBigFilesFromDirectory(%s) returned FALSE!", TheGlobalData->m_modDir.str())); + ArchiveMountEntry entry; + AsciiString reason; + if (!parseMountEntry(AsciiString(includePaths[i].c_str()), &entry, &reason)) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Data pack layer %s: path ignored (%s): %s", layer, reason.str(), includePaths[i].c_str()); + continue; + } + mountListedArchive(entry, priority + static_cast(i), excludes); } } @@ -306,15 +504,15 @@ ArchiveFileSystem::ArchivedDirectoryInfoResult ArchiveFileSystem::getArchivedDir AsciiString token; AsciiString tokenizer = directory; tokenizer.toLower(); - Bool infoInPath = tokenizer.nextToken(&token, "\\/"); + tokenizer.nextToken(&token, "\\/"); - while (infoInPath && (!token.find('.') || tokenizer.find('.'))) + while (!tokenizer.isEmpty()) { ArchivedDirectoryInfoMap::iterator tempiter = dirInfo->m_directories.find(token); if (tempiter != dirInfo->m_directories.end()) { dirInfo = &tempiter->second; - infoInPath = tokenizer.nextToken(&token, "\\/"); + tokenizer.nextToken(&token, "\\/"); } else { diff --git a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp b/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp similarity index 54% rename from Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp rename to Core/GameEngine/Source/Common/System/BIGFileSystem.cpp index b824dcaed57..6919947f36a 100644 --- a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFileSystem.cpp +++ b/Core/GameEngine/Source/Common/System/BIGFileSystem.cpp @@ -22,13 +22,16 @@ // // //////////////////////////////////////////////////////////////////////////////// -//////// Win32BIGFileSystem.h /////////////////////////// +//////// BIGFileSystem.cpp /////////////////////////// // Bryan Cleveland, August 2002 ///////////////////////////////////////////////////////////// +#include "PreRTS.h" + +#include "Common/BIGFileSystem.h" + #include "Common/AudioAffect.h" #include "Common/ArchiveFile.h" -#include "Common/ArchiveFileSystem.h" #include "Common/file.h" #include "Common/GameAudio.h" #include "Common/GameMemory.h" @@ -38,48 +41,56 @@ #include "Common/Registry.h" #endif -#include "Win32Device/Common/Win32BIGFile.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" #include "Utility/endian_compat.h" +#include "../NGMP_include.h" +#include "../OnlineServices_Init.h" static const char *BIGFileIdentifier = "BIGF"; -Win32BIGFileSystem::Win32BIGFileSystem() : ArchiveFileSystem() { +BIGFileSystem::BIGFileSystem() : ArchiveFileSystem() { } -Win32BIGFileSystem::~Win32BIGFileSystem() { +BIGFileSystem::~BIGFileSystem() { } -void Win32BIGFileSystem::init() { +void BIGFileSystem::init() { DEBUG_ASSERTCRASH(TheLocalFileSystem != nullptr, ("TheLocalFileSystem must be initialized before TheArchiveFileSystem.")); if (TheLocalFileSystem == nullptr) { return; } - loadBigFilesFromDirectory("", "*.big"); + loadArchiveFilesFromDirectory("", "*.big"); #if RTS_ZEROHOUR // load original Generals assets + // The Steam install ships them in ZH_Generals, which the scan above already mounted. Mounting the Generals install as well would only add its mods. + if (TheLocalFileSystem->doesFileExist("ZH_Generals\\INI.big")) { + m_generalsBaseSource = "ZH_Generals folder"; + return; + } + AsciiString installPath; GetStringFromGeneralsRegistry("", "InstallPath", installPath ); //@todo this will need to be ramped up to a crash for release DEBUG_ASSERTCRASH(!installPath.isEmpty(), ("Be 1337! Go install Generals!")); - if (!installPath.isEmpty()) - loadBigFilesFromDirectory(installPath, "*.big"); + if (!installPath.isEmpty()) { + m_generalsBaseSource.format("Generals install folder %s", installPath.str()); + loadArchiveFilesFromDirectory(installPath, "*.big"); + } #endif } -void Win32BIGFileSystem::reset() { +void BIGFileSystem::reset() { } -void Win32BIGFileSystem::update() { +void BIGFileSystem::update() { } -void Win32BIGFileSystem::postProcessLoad() { +void BIGFileSystem::postProcessLoad() { } -ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { +ArchiveFile * BIGFileSystem::openArchiveFile(const Char *filename) { File *fp = TheLocalFileSystem->openFile(filename, File::READ | File::BINARY); AsciiString archiveFileName; archiveFileName = filename; @@ -87,7 +98,7 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { Int archiveFileSize = 0; Int numLittleFiles = 0; - DEBUG_LOG(("Win32BIGFileSystem::openArchiveFile - opening BIG file %s", filename)); + DEBUG_LOG(("BIGFileSystem::openArchiveFile - opening BIG file %s", filename)); if (fp == nullptr) { DEBUG_CRASH(("Could not open archive file %s for parsing", filename)); @@ -113,7 +124,7 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { return nullptr; } - DEBUG_LOG(("Win32BIGFileSystem::openArchiveFile - size of archive file is %d bytes", archiveFileSize)); + DEBUG_LOG(("BIGFileSystem::openArchiveFile - size of archive file is %d bytes", archiveFileSize)); // char t; @@ -127,7 +138,7 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { } numLittleFiles = betoh(numLittleFiles); - DEBUG_LOG(("Win32BIGFileSystem::openArchiveFile - %d are contained in archive", numLittleFiles)); + DEBUG_LOG(("BIGFileSystem::openArchiveFile - %d are contained in archive", numLittleFiles)); // for (Int i = 0; i < 2; ++i) { // t = buffer[i]; // buffer[i] = buffer[(4-i)-1]; @@ -137,9 +148,8 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { // seek to the beginning of the directory listing. fp->seek(0x10, File::START); // read in each directory listing. - ArchivedFileInfo *fileInfo = NEW ArchivedFileInfo; - // TheSuperHackers @fix Mauller 23/04/2025 Create new file handle when necessary to prevent memory leak - ArchiveFile *archiveFile = NEW Win32BIGFile(filename, AsciiString::TheEmptyString); + ArchivedFileInfo fileInfo; + ArchiveFile *archiveFile = NEW ArchiveFile(filename, AsciiString::TheEmptyString); Bool corrupt = FALSE; for (Int i = 0; i < numLittleFiles; ++i) { @@ -153,9 +163,9 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { filesize = betoh(filesize); fileOffset = betoh(fileOffset); - fileInfo->m_archiveFilename = archiveFileName; - fileInfo->m_offset = fileOffset; - fileInfo->m_size = filesize; + fileInfo.m_archiveFilename = archiveFileName; + fileInfo.m_offset = fileOffset; + fileInfo.m_size = filesize; // read in the path name of the file. // the name must terminate within the buffer @@ -179,8 +189,8 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { --filenameIndex; } - fileInfo->m_filename = (char *)(buffer + filenameIndex + 1); - fileInfo->m_filename.toLower(); + fileInfo.m_filename = (char *)(buffer + filenameIndex + 1); + fileInfo.m_filename.toLower(); buffer[filenameIndex + 1] = 0; AsciiString path; @@ -188,16 +198,15 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { AsciiString debugpath; debugpath = path; - debugpath.concat(fileInfo->m_filename); -// DEBUG_LOG(("Win32BIGFileSystem::openArchiveFile - adding file %s to archive file %s, file number %d", debugpath.str(), fileInfo->m_archiveFilename.str(), i)); + debugpath.concat(fileInfo.m_filename); +// DEBUG_LOG(("BIGFileSystem::openArchiveFile - adding file %s to archive file %s, file number %d", debugpath.str(), fileInfo->m_archiveFilename.str(), i)); - archiveFile->addFile(path, fileInfo); + archiveFile->addFile(path, &fileInfo); } if (corrupt) { DEBUG_CRASH(("Truncated or corrupt directory in BIG file %s", filename)); delete archiveFile; - delete fileInfo; fp->close(); fp = nullptr; return nullptr; @@ -205,15 +214,12 @@ ArchiveFile * Win32BIGFileSystem::openArchiveFile(const Char *filename) { archiveFile->attachFile(fp); - delete fileInfo; - fileInfo = nullptr; - // leave fp open as the archive file will be using it. return archiveFile; } -void Win32BIGFileSystem::closeArchiveFile(const Char *filename) { +void BIGFileSystem::closeArchiveFile(const Char *filename) { // Need to close the specified big file ArchiveFileMap::iterator it = m_archiveFileMap.find(filename); if (it == m_archiveFileMap.end()) { @@ -234,13 +240,13 @@ void Win32BIGFileSystem::closeArchiveFile(const Char *filename) { m_archiveFileMap.erase(it); } -void Win32BIGFileSystem::closeAllArchiveFiles() { +void BIGFileSystem::closeAllArchiveFiles() { } -void Win32BIGFileSystem::closeAllFiles() { +void BIGFileSystem::closeAllFiles() { } -Bool Win32BIGFileSystem::loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite) { +Bool BIGFileSystem::loadArchiveFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite) { FilenameList filenameList; TheLocalFileSystem->getFileListInDirectory(dir, "", fileMask, filenameList, TRUE); @@ -258,13 +264,25 @@ Bool Win32BIGFileSystem::loadBigFilesFromDirectory(AsciiString dir, AsciiString } #endif +#if RTS_ZEROHOUR + const Bool zeroHour = TRUE; +#else + const Bool zeroHour = FALSE; +#endif + // The first scan is the game folder, the second one the Generals install: the retail archives are the bottom of the lookup + const Int priority = overwrite ? getNextModPriority() : getGameFolderArchivePriority((*it), dir.isNotEmpty(), zeroHour); + if (priority == ARCHIVE_PRIORITY_EXCLUDED) { + it++; + continue; + } + ArchiveFile *archiveFile = openArchiveFile((*it).str()); if (archiveFile != nullptr) { - DEBUG_LOG(("Win32BIGFileSystem::loadBigFilesFromDirectory - loading %s into the directory tree.", (*it).str())); - loadIntoDirectoryTree(archiveFile, overwrite); + DEBUG_LOG(("BIGFileSystem::loadArchiveFilesFromDirectory - loading %s into the directory tree.", (*it).str())); + loadIntoDirectoryTree(archiveFile, priority); m_archiveFileMap[(*it)] = archiveFile; - DEBUG_LOG(("Win32BIGFileSystem::loadBigFilesFromDirectory - %s inserted into the archive file map.", (*it).str())); + DEBUG_LOG(("BIGFileSystem::loadArchiveFilesFromDirectory - %s inserted into the archive file map.", (*it).str())); actuallyAdded = TRUE; } @@ -273,3 +291,46 @@ Bool Win32BIGFileSystem::loadBigFilesFromDirectory(AsciiString dir, AsciiString return actuallyAdded; } + +void BIGFileSystem::loadMods() +{ +#if defined(GENERALS_ONLINE) + // Order, highest first: loose files of the game folder, -mod and -modDir, the BIGs of the game folders that are not retail (user.exclude_paths removes some), + // the data pack layers of settings.json (user, generals_online, community_patch), the retail archives. The BIGs of the game folders were mounted by init(). + NetworkLog(ELogVerbosity::LOG_RELEASE, "Generals data: %s", m_generalsBaseSource.isEmpty() ? "none" : m_generalsBaseSource.str()); + NetworkLog(ELogVerbosity::LOG_RELEASE, "Game folder archives above the data pack layers: %d", static_cast(m_gameFolderArchives.size())); + for (const AsciiString& gameFolderArchive : m_gameFolderArchives) + NetworkLog(ELogVerbosity::LOG_RELEASE, " %s", gameFolderArchive.str()); + + const GenOnlineSettings& settings = NGMP_OnlineServicesManager::Settings; + mountDataPackLayer("user", settings.DataPacks_User().m_enabled, settings.DataPacks_User().m_includePaths, &TheFileSystem->getExcludedPaths(), ARCHIVE_PRIORITY_USER); + mountDataPackLayer("generals_online", settings.DataPacks_GeneralsOnline().m_enabled, settings.DataPacks_GeneralsOnline().m_includePaths, nullptr, ARCHIVE_PRIORITY_GENERALS_ONLINE); + + // -disableCommunityDataPatch skips the community_patch layer for this launch only, the setting is not changed + const Bool communityPatchEnabled = settings.DataPacks_CommunityPatch().m_enabled && !TheGlobalData->m_commandLineData.isCommunityDataPatchDisabled(); + mountDataPackLayer("community_patch", communityPatchEnabled, settings.DataPacks_CommunityPatch().m_includePaths, nullptr, ARCHIVE_PRIORITY_COMMUNITY_PATCH); +#endif + + if (TheGlobalData->m_modBIG.isNotEmpty()) + { + ArchiveFile *archiveFile = openArchiveFile(TheGlobalData->m_modBIG.str()); + + if (archiveFile != nullptr) { + DEBUG_LOG(("BIGFileSystem::loadMods - loading %s into the directory tree.", TheGlobalData->m_modBIG.str())); + loadIntoDirectoryTree(archiveFile, getNextModPriority()); + m_archiveFileMap[TheGlobalData->m_modBIG] = archiveFile; + DEBUG_LOG(("BIGFileSystem::loadMods - %s inserted into the archive file map.", TheGlobalData->m_modBIG.str())); + } + else + { + DEBUG_LOG(("BIGFileSystem::loadMods - could not openArchiveFile(%s)", TheGlobalData->m_modBIG.str())); + } + } + + if (TheGlobalData->m_modDir.isNotEmpty()) + { + MAYBE_UNUSED Bool ret = loadArchiveFilesFromDirectory(TheGlobalData->m_modDir, "*.big", TRUE); + (void)ret; + DEBUG_ASSERTLOG(ret, ("loadArchiveFilesFromDirectory(%s) returned FALSE!", TheGlobalData->m_modDir.str())); + } +} diff --git a/Core/GameEngine/Source/Common/System/FileSystem.cpp b/Core/GameEngine/Source/Common/System/FileSystem.cpp index b8e4c4695b6..e4df60933a4 100644 --- a/Core/GameEngine/Source/Common/System/FileSystem.cpp +++ b/Core/GameEngine/Source/Common/System/FileSystem.cpp @@ -56,6 +56,8 @@ #include "Lib/PathUtil.h" +#include + DECLARE_PERF_TIMER(FileSystem) @@ -118,6 +120,184 @@ FileSystem *TheFileSystem = nullptr; //---------------------------------------------------------------------------- +//============================================================================ +// PathExcludeList +//============================================================================ + +static std::string normalizeExcludeText(const std::string& text) +{ + std::string value = text; + for (size_t i = 0; i < value.size(); ++i) + { + if (value[i] == '/') + value[i] = '\\'; + value[i] = static_cast(tolower(static_cast(value[i]))); + } + return value; +} + +static Bool isAbsoluteExcludePath(const std::string& value) +{ + return (value.size() > 2 && isalpha(static_cast(value[0])) && value[1] == ':' && value[2] == '\\') + || (value.size() > 2 && value[0] == '\\' && value[1] == '\\'); +} + +static std::string stripCurrentFolder(const std::string& value) +{ + size_t start = 0; + while (start + 2 <= value.size() && value.compare(start, 2, ".\\") == 0) + start += 2; + return value.substr(start); +} + +Bool PathExcludeList::parse(const std::string& text, std::string* pattern, AsciiString* reason) +{ + std::string value = text; + while (!value.empty() && isspace(static_cast(value[value.size() - 1]))) + value.erase(value.size() - 1); + size_t start = 0; + while (start < value.size() && isspace(static_cast(value[start]))) + ++start; + value = normalizeExcludeText(value.substr(start)); + + if (value.empty()) + { + if (reason != nullptr) + *reason = "empty"; + return FALSE; + } + for (size_t i = 0; i < value.size(); ++i) + { + const unsigned char c = static_cast(value[i]); + if (c < 32 || c == '<' || c == '>' || c == '"' || c == '|') + { + if (reason != nullptr) + *reason = "invalid character"; + return FALSE; + } + } + + std::string rest; + std::string prefix; + if (value.compare(0, 5, "game:") == 0 || value.compare(0, 5, "user:") == 0) + { + prefix = value.substr(0, 5); + rest = stripCurrentFolder(value.substr(5)); + if (rest.empty() || rest[0] == '\\' || rest.find(':') != std::string::npos) + { + if (reason != nullptr) + *reason = "invalid path"; + return FALSE; + } + } + else if (isAbsoluteExcludePath(value)) + { + rest = value; + } + else + { + if (reason != nullptr) + *reason = "needs the prefix game: or user:, or an absolute path"; + return FALSE; + } + + // no parent folders + size_t pos = 0; + while (pos <= rest.size()) + { + size_t next = rest.find('\\', pos); + if (next == std::string::npos) + next = rest.size(); + if (next - pos == 2 && rest.compare(pos, 2, "..") == 0) + { + if (reason != nullptr) + *reason = "parent folder"; + return FALSE; + } + pos = next + 1; + } + + // a folder: everything in it + if (rest[rest.size() - 1] == '\\') + rest += "*"; + + if (pattern != nullptr) + *pattern = prefix + rest; + return TRUE; +} + +std::string PathExcludeList::makeKey(const Char* path) +{ + std::string value = stripCurrentFolder(normalizeExcludeText(path)); + if (isAbsoluteExcludePath(value)) + return value; + return "game:" + value; +} + +std::string PathExcludeList::makeKey(const Char* path, const Char* userDataPath) +{ + std::string value = normalizeExcludeText(path); + std::string userData = normalizeExcludeText(userDataPath); + if (!userData.empty() && userData[userData.size() - 1] != '\\') + userData += "\\"; + if (!userData.empty() && value.size() > userData.size() && value.compare(0, userData.size(), userData) == 0) + return "user:" + value.substr(userData.size()); + return makeKey(path); +} + +void PathExcludeList::set(const std::vector& entries) +{ + m_patterns.clear(); + for (size_t i = 0; i < entries.size(); ++i) + { + std::string pattern; + if (parse(entries[i], &pattern, nullptr)) + m_patterns.push_back(pattern); + } +} + +// * matches any text, ? one character +static Bool matchesExcludePattern(const char* pattern, const char* text) +{ + const char* starPattern = nullptr; + const char* starText = nullptr; + while (*text != '\0') + { + if (*pattern == '*') + { + starPattern = ++pattern; + starText = text; + } + else if (*pattern == '?' || *pattern == *text) + { + ++pattern; + ++text; + } + else if (starPattern != nullptr) + { + pattern = starPattern; + text = ++starText; + } + else + { + return FALSE; + } + } + while (*pattern == '*') + ++pattern; + return *pattern == '\0'; +} + +Bool PathExcludeList::matches(const std::string& key) const +{ + for (size_t i = 0; i < m_patterns.size(); ++i) + { + if (matchesExcludePattern(m_patterns[i].c_str(), key.c_str())) + return TRUE; + } + return FALSE; +} + //============================================================================ // FileSystem::FileSystem //============================================================================ @@ -179,14 +359,17 @@ File* FileSystem::openFile( const Char *filename, Int access, size_t bufferSize if ( TheLocalFileSystem != nullptr ) { + // an excluded loose file does not exist, unless the file is written + const Bool excluded = (access & (File::WRITE | File::APPEND | File::CREATE | File::TRUNCATE)) == 0 && isLooseFileExcluded(filename); + if (instance != 0) { - if (TheLocalFileSystem->doesFileExist(filename)) + if (!excluded && TheLocalFileSystem->doesFileExist(filename)) { --instance; } } - else + else if (!excluded) { file = TheLocalFileSystem->openFile( filename, access, bufferSize ); @@ -242,7 +425,7 @@ Bool FileSystem::doesFileExist(const Char *filename, FileInstance instance) cons } #endif - if (TheLocalFileSystem->doesFileExist(filename)) + if (!isLooseFileExcluded(filename) && TheLocalFileSystem->doesFileExist(filename)) { if (instance == 0) { @@ -286,7 +469,20 @@ Bool FileSystem::doesFileExist(const Char *filename, FileInstance instance) cons void FileSystem::getFileListInDirectory(const AsciiString& directory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const { USE_PERF_TIMER(FileSystem) - TheLocalFileSystem->getFileListInDirectory(AsciiString::TheEmptyString, directory, searchName, filenameList, searchSubdirectories); + if (m_excludedPaths.isEmpty()) + { + TheLocalFileSystem->getFileListInDirectory(AsciiString::TheEmptyString, directory, searchName, filenameList, searchSubdirectories); + } + else + { + FilenameList localFiles; + TheLocalFileSystem->getFileListInDirectory(AsciiString::TheEmptyString, directory, searchName, localFiles, searchSubdirectories); + for (FilenameListIter it = localFiles.begin(); it != localFiles.end(); ++it) + { + if (!isLooseFileExcluded(it->str())) + filenameList.insert(*it); + } + } TheArchiveFileSystem->getFileListInDirectory(AsciiString::TheEmptyString, directory, searchName, filenameList, searchSubdirectories); } @@ -304,7 +500,7 @@ Bool FileSystem::getFileInfo(const AsciiString& filename, FileInfo *fileInfo, Fi } memset(fileInfo, 0, sizeof(*fileInfo)); - if (TheLocalFileSystem->getFileInfo(filename, fileInfo)) { + if (!isLooseFileExcluded(filename.str()) && TheLocalFileSystem->getFileInfo(filename, fileInfo)) { if (instance == 0) { return TRUE; } diff --git a/Core/GameEngineDevice/CMakeLists.txt b/Core/GameEngineDevice/CMakeLists.txt index 6added5403b..71933e7c69f 100644 --- a/Core/GameEngineDevice/CMakeLists.txt +++ b/Core/GameEngineDevice/CMakeLists.txt @@ -83,8 +83,6 @@ set(GAMEENGINEDEVICE_SRC # Include/W3DDevice/GameLogic/W3DGameLogic.h # Include/W3DDevice/GameLogic/W3DGhostObject.h # Include/W3DDevice/GameLogic/W3DTerrainLogic.h - Include/Win32Device/Common/Win32BIGFile.h - Include/Win32Device/Common/Win32BIGFileSystem.h # Include/Win32Device/Common/Win32GameEngine.h Include/Win32Device/Common/Win32LocalFile.h Include/Win32Device/Common/Win32LocalFileSystem.h @@ -186,8 +184,6 @@ set(GAMEENGINEDEVICE_SRC # Source/W3DDevice/GameLogic/W3DGameLogic.cpp # Source/W3DDevice/GameLogic/W3DGhostObject.cpp # Source/W3DDevice/GameLogic/W3DTerrainLogic.cpp - Source/Win32Device/Common/Win32BIGFile.cpp - Source/Win32Device/Common/Win32BIGFileSystem.cpp # Source/Win32Device/Common/Win32GameEngine.cpp Source/Win32Device/Common/Win32LocalFile.cpp Source/Win32Device/Common/Win32LocalFileSystem.cpp @@ -200,12 +196,8 @@ set(GAMEENGINEDEVICE_SRC # Add C++ 17 FileSystem implementation for non-VS6 builds if(NOT IS_VS6_BUILD) list(APPEND GAMEENGINEDEVICE_SRC - Include/StdDevice/Common/StdBIGFile.h - Include/StdDevice/Common/StdBIGFileSystem.h Include/StdDevice/Common/StdLocalFile.h Include/StdDevice/Common/StdLocalFileSystem.h - Source/StdDevice/Common/StdBIGFile.cpp - Source/StdDevice/Common/StdBIGFileSystem.cpp Source/StdDevice/Common/StdLocalFile.cpp Source/StdDevice/Common/StdLocalFileSystem.cpp ) diff --git a/Core/GameEngineDevice/Include/StdDevice/Common/StdBIGFile.h b/Core/GameEngineDevice/Include/StdDevice/Common/StdBIGFile.h deleted file mode 100644 index 36538853352..00000000000 --- a/Core/GameEngineDevice/Include/StdDevice/Common/StdBIGFile.h +++ /dev/null @@ -1,53 +0,0 @@ -/* -** Command & Conquer Generals Zero Hour(tm) -** Copyright 2025 Electronic Arts Inc. -** -** This program is free software: you can redistribute it and/or modify -** it under the terms of the GNU General Public License as published by -** the Free Software Foundation, either version 3 of the License, or -** (at your option) any later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program. If not, see . -*/ - -//////////////////////////////////////////////////////////////////////////////// -// // -// (c) 2001-2003 Electronic Arts Inc. // -// // -//////////////////////////////////////////////////////////////////////////////// - -/////// StdBIGFile.h //////////////////////////////////// -// Stephan Vedder, April 2025 -/////////////////////////////////////////////////////////// - -#pragma once - -#include "Common/ArchiveFile.h" -#include "Common/AsciiString.h" -#include "Common/List.h" - -class StdBIGFile : public ArchiveFile -{ - public: - StdBIGFile(AsciiString name, AsciiString path); - virtual ~StdBIGFile() override; - - virtual Bool getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const override; ///< fill in the fileInfo struct with info about the requested file. - virtual File* openFile( const Char *filename, Int access = 0 ) override;///< Open the specified file within the BIG file - virtual void closeAllFiles() override; ///< Close all file opened in this BIG file - virtual AsciiString getName() override; ///< Returns the name of the BIG file - virtual AsciiString getPath() override; ///< Returns full path and name of BIG file - virtual void setSearchPriority( Int new_priority ) override; ///< Set this BIG file's search priority - virtual void close() override; ///< Close this BIG file - - protected: - - AsciiString m_name; ///< BIG file name - AsciiString m_path; ///< BIG file path -}; diff --git a/Core/GameEngineDevice/Include/StdDevice/Common/StdBIGFileSystem.h b/Core/GameEngineDevice/Include/StdDevice/Common/StdBIGFileSystem.h deleted file mode 100644 index b8cb316ab7c..00000000000 --- a/Core/GameEngineDevice/Include/StdDevice/Common/StdBIGFileSystem.h +++ /dev/null @@ -1,55 +0,0 @@ -/* -** Command & Conquer Generals Zero Hour(tm) -** Copyright 2025 Electronic Arts Inc. -** -** This program is free software: you can redistribute it and/or modify -** it under the terms of the GNU General Public License as published by -** the Free Software Foundation, either version 3 of the License, or -** (at your option) any later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program. If not, see . -*/ - -//////////////////////////////////////////////////////////////////////////////// -// // -// (c) 2001-2003 Electronic Arts Inc. // -// // -//////////////////////////////////////////////////////////////////////////////// - -//////// StdBIGFileSystem.h /////////////////////////// -// Stephan Vedder, April 2025 -///////////////////////////////////////////////////////////// - -#pragma once - -#include "Common/ArchiveFileSystem.h" - -class StdBIGFileSystem : public ArchiveFileSystem -{ -public: - StdBIGFileSystem(); - virtual ~StdBIGFileSystem() override; - - virtual void init() override; - virtual void update() override; - virtual void reset() override; - virtual void postProcessLoad() override; - - // ArchiveFile operations - virtual void closeAllArchiveFiles() override; ///< Close all Archivefiles currently open - - // File operations - virtual ArchiveFile * openArchiveFile(const Char *filename) override; - virtual void closeArchiveFile(const Char *filename) override; - virtual void closeAllFiles() override; ///< Close all files associated with ArchiveFiles - - virtual Bool loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite = FALSE) override; -protected: - -}; diff --git a/Core/GameEngineDevice/Include/Win32Device/Common/Win32BIGFile.h b/Core/GameEngineDevice/Include/Win32Device/Common/Win32BIGFile.h deleted file mode 100644 index 093aadbcf4c..00000000000 --- a/Core/GameEngineDevice/Include/Win32Device/Common/Win32BIGFile.h +++ /dev/null @@ -1,53 +0,0 @@ -/* -** Command & Conquer Generals Zero Hour(tm) -** Copyright 2025 Electronic Arts Inc. -** -** This program is free software: you can redistribute it and/or modify -** it under the terms of the GNU General Public License as published by -** the Free Software Foundation, either version 3 of the License, or -** (at your option) any later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program. If not, see . -*/ - -//////////////////////////////////////////////////////////////////////////////// -// // -// (c) 2001-2003 Electronic Arts Inc. // -// // -//////////////////////////////////////////////////////////////////////////////// - -/////// Win32BIGFile.h //////////////////////////////////// -// Bryan Cleveland, August 2002 -/////////////////////////////////////////////////////////// - -#pragma once - -#include "Common/ArchiveFile.h" -#include "Common/AsciiString.h" -#include "Common/List.h" - -class Win32BIGFile : public ArchiveFile -{ - public: - Win32BIGFile(AsciiString name, AsciiString path); - virtual ~Win32BIGFile() override; - - virtual Bool getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const override; ///< fill in the fileInfo struct with info about the requested file. - virtual File* openFile( const Char *filename, Int access = 0 ) override;///< Open the specified file within the BIG file - virtual void closeAllFiles() override; ///< Close all file opened in this BIG file - virtual AsciiString getName() override; ///< Returns the name of the BIG file - virtual AsciiString getPath() override; ///< Returns full path and name of BIG file - virtual void setSearchPriority( Int new_priority ) override; ///< Set this BIG file's search priority - virtual void close() override; ///< Close this BIG file - - protected: - - AsciiString m_name; ///< BIG file name - AsciiString m_path; ///< BIG file path -}; diff --git a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFile.cpp b/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFile.cpp deleted file mode 100644 index c873b733524..00000000000 --- a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFile.cpp +++ /dev/null @@ -1,168 +0,0 @@ -/* -** Command & Conquer Generals Zero Hour(tm) -** Copyright 2025 Electronic Arts Inc. -** -** This program is free software: you can redistribute it and/or modify -** it under the terms of the GNU General Public License as published by -** the Free Software Foundation, either version 3 of the License, or -** (at your option) any later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program. If not, see . -*/ - -//////////////////////////////////////////////////////////////////////////////// -// // -// (c) 2001-2003 Electronic Arts Inc. // -// // -//////////////////////////////////////////////////////////////////////////////// - -////// StdBIGFile.cpp ///////////////////////// -// Stephan Vedder, April 2025 -///////////////////////////////////////////////////// - -#include "Common/LocalFile.h" -#include "Common/LocalFileSystem.h" -#include "Common/RAMFile.h" -#include "Common/StreamingArchiveFile.h" -#include "Common/GameMemory.h" -#include "Common/PerfTimer.h" -#include "StdDevice/Common/StdBIGFile.h" - -//============================================================================ -// StdBIGFile::StdBIGFile -//============================================================================ - -StdBIGFile::StdBIGFile(AsciiString name, AsciiString path) - : m_name(name) - , m_path(path) -{ - -} - -//============================================================================ -// StdBIGFile::~StdBIGFile -//============================================================================ - -StdBIGFile::~StdBIGFile() -{ - -} - -//============================================================================ -// StdBIGFile::openFile -//============================================================================ - -File* StdBIGFile::openFile( const Char *filename, Int access ) -{ - const ArchivedFileInfo *fileInfo = getArchivedFileInfo(AsciiString(filename)); - - if (fileInfo == nullptr) { - return nullptr; - } - - RAMFile *ramFile = nullptr; - - if (BitIsSet(access, File::STREAMING)) - ramFile = newInstance( StreamingArchiveFile ); - else - ramFile = newInstance( RAMFile ); - - ramFile->deleteOnClose(); - if (ramFile->openFromArchive(m_file, fileInfo->m_filename, fileInfo->m_offset, fileInfo->m_size) == FALSE) { - ramFile->close(); - ramFile = nullptr; - return nullptr; - } - - if ((access & File::WRITE) == 0) { - // requesting read only access. Just return the RAM file. - return ramFile; - } - - // whoever is opening this file wants write access, so copy the file to the local disk - // and return that file pointer. - - constexpr size_t bufferSize = 0; - File *localFile = TheLocalFileSystem->openFile(filename, access, bufferSize); - if (localFile != nullptr) { - ramFile->copyDataToFile(localFile); - } - - ramFile->close(); - ramFile = nullptr; - - return localFile; -} - -//============================================================================ -// StdBIGFile::closeAllFiles -//============================================================================ - -void StdBIGFile::closeAllFiles() -{ - -} - -//============================================================================ -// StdBIGFile::getName -//============================================================================ - -AsciiString StdBIGFile::getName() -{ - return m_name; -} - -//============================================================================ -// StdBIGFile::getPath -//============================================================================ - -AsciiString StdBIGFile::getPath() -{ - return m_path; -} - -//============================================================================ -// StdBIGFile::setSearchPriority -//============================================================================ - -void StdBIGFile::setSearchPriority( Int new_priority ) -{ - -} - -//============================================================================ -// StdBIGFile::close -//============================================================================ - -void StdBIGFile::close() -{ - -} - -//============================================================================ -// StdBIGFile::getFileInfo -//============================================================================ - -Bool StdBIGFile::getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const -{ - const ArchivedFileInfo *tempFileInfo = getArchivedFileInfo(filename); - - if (tempFileInfo == nullptr) { - return FALSE; - } - - TheLocalFileSystem->getFileInfo(AsciiString(m_file->getName()), fileInfo); - - // fill in the size info. Since the size can't be bigger than a JUNK file, the high Int will always be 0. - fileInfo->sizeHigh = 0; - fileInfo->sizeLow = tempFileInfo->m_size; - - return TRUE; -} - diff --git a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp b/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp deleted file mode 100644 index ffd0150f4c5..00000000000 --- a/Core/GameEngineDevice/Source/StdDevice/Common/StdBIGFileSystem.cpp +++ /dev/null @@ -1,242 +0,0 @@ -/* -** Command & Conquer Generals Zero Hour(tm) -** Copyright 2025 Electronic Arts Inc. -** -** This program is free software: you can redistribute it and/or modify -** it under the terms of the GNU General Public License as published by -** the Free Software Foundation, either version 3 of the License, or -** (at your option) any later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program. If not, see . -*/ - -//////////////////////////////////////////////////////////////////////////////// -// // -// (c) 2001-2003 Electronic Arts Inc. // -// // -//////////////////////////////////////////////////////////////////////////////// - -//////// StdBIGFileSystem.h /////////////////////////// -// Stephan Vedder, April 2025 -///////////////////////////////////////////////////////////// - -#include "Common/AudioAffect.h" -#include "Common/ArchiveFile.h" -#include "Common/ArchiveFileSystem.h" -#include "Common/file.h" -#include "Common/GameAudio.h" -#include "Common/GameMemory.h" -#include "Common/LocalFileSystem.h" - -#if RTS_ZEROHOUR -#include "Common/Registry.h" -#endif - -#include "StdDevice/Common/StdBIGFile.h" -#include "StdDevice/Common/StdBIGFileSystem.h" -#include "Utility/endian_compat.h" - -static const char *BIGFileIdentifier = "BIGF"; - -StdBIGFileSystem::StdBIGFileSystem() : ArchiveFileSystem() { -} - -StdBIGFileSystem::~StdBIGFileSystem() { -} - -void StdBIGFileSystem::init() { - DEBUG_ASSERTCRASH(TheLocalFileSystem != nullptr, ("TheLocalFileSystem must be initialized before TheArchiveFileSystem.")); - if (TheLocalFileSystem == nullptr) { - return; - } - - loadBigFilesFromDirectory("", "*.big"); - -#if RTS_ZEROHOUR - // load original Generals assets - AsciiString installPath; - GetStringFromGeneralsRegistry("", "InstallPath", installPath ); - //@todo this will need to be ramped up to a crash for release - DEBUG_ASSERTCRASH(!installPath.isEmpty(), ("Be 1337! Go install Generals!")); - if (!installPath.isEmpty()) - loadBigFilesFromDirectory(installPath, "*.big"); -#endif -} - -void StdBIGFileSystem::reset() { -} - -void StdBIGFileSystem::update() { -} - -void StdBIGFileSystem::postProcessLoad() { -} - -ArchiveFile * StdBIGFileSystem::openArchiveFile(const Char *filename) { - File *fp = TheLocalFileSystem->openFile(filename, File::READ | File::BINARY); - AsciiString archiveFileName; - archiveFileName = filename; - archiveFileName.toLower(); - Int archiveFileSize = 0; - Int numLittleFiles = 0; - - ArchiveFile *archiveFile = NEW StdBIGFile(filename, AsciiString::TheEmptyString); - - DEBUG_LOG(("StdBIGFileSystem::openArchiveFile - opening BIG file %s", filename)); - - if (fp == nullptr) { - DEBUG_CRASH(("Could not open archive file %s for parsing", filename)); - return nullptr; - } - - AsciiString asciibuf; - char buffer[_MAX_PATH]; - fp->read(buffer, 4); // read the "BIG" at the beginning of the file. - buffer[4] = 0; - if (strcmp(buffer, BIGFileIdentifier) != 0) { - DEBUG_CRASH(("Error reading BIG file identifier in file %s", filename)); - fp->close(); - fp = nullptr; - return nullptr; - } - - // read in the file size. - fp->read(&archiveFileSize, 4); - - DEBUG_LOG(("StdBIGFileSystem::openArchiveFile - size of archive file is %d bytes", archiveFileSize)); - -// char t; - - // read in the number of files contained in this BIG file. - // change the order of the bytes cause the file size is in reverse byte order for some reason. - fp->read(&numLittleFiles, 4); - numLittleFiles = betoh(numLittleFiles); - - DEBUG_LOG(("StdBIGFileSystem::openArchiveFile - %d are contained in archive", numLittleFiles)); -// for (Int i = 0; i < 2; ++i) { -// t = buffer[i]; -// buffer[i] = buffer[(4-i)-1]; -// buffer[(4-i)-1] = t; -// } - - // seek to the beginning of the directory listing. - fp->seek(0x10, File::START); - // read in each directory listing. - ArchivedFileInfo *fileInfo = NEW ArchivedFileInfo; - - for (Int i = 0; i < numLittleFiles; ++i) { - Int filesize = 0; - Int fileOffset = 0; - fp->read(&fileOffset, 4); - fp->read(&filesize, 4); - - filesize = betoh(filesize); - fileOffset = betoh(fileOffset); - - fileInfo->m_archiveFilename = archiveFileName; - fileInfo->m_offset = fileOffset; - fileInfo->m_size = filesize; - - // read in the path name of the file. - Int pathIndex = -1; - do { - ++pathIndex; - fp->read(buffer + pathIndex, 1); - } while (buffer[pathIndex] != 0); - - Int filenameIndex = pathIndex; - while ((filenameIndex >= 0) && (buffer[filenameIndex] != '\\') && (buffer[filenameIndex] != '/')) { - --filenameIndex; - } - - fileInfo->m_filename = (char *)(buffer + filenameIndex + 1); - fileInfo->m_filename.toLower(); - buffer[filenameIndex + 1] = 0; - - AsciiString path; - path = buffer; - - AsciiString debugpath; - debugpath = path; - debugpath.concat(fileInfo->m_filename); -// DEBUG_LOG(("StdBIGFileSystem::openArchiveFile - adding file %s to archive file %s, file number %d", debugpath.str(), fileInfo->m_archiveFilename.str(), i)); - - archiveFile->addFile(path, fileInfo); - } - - archiveFile->attachFile(fp); - - delete fileInfo; - fileInfo = nullptr; - - // leave fp open as the archive file will be using it. - - return archiveFile; -} - -void StdBIGFileSystem::closeArchiveFile(const Char *filename) { - // Need to close the specified big file - ArchiveFileMap::iterator it = m_archiveFileMap.find(filename); - if (it == m_archiveFileMap.end()) { - return; - } - - if (stricmp(filename, MUSIC_BIG) == 0) { - // Stop the current audio - TheAudio->stopAudio(AudioAffect_Music); - - // No need to turn off other audio, as the lookups will just fail. - } - DEBUG_ASSERTCRASH(stricmp(filename, MUSIC_BIG) == 0, ("Attempting to close Archive file '%s', need to add code to handle its shutdown correctly.", filename)); - - // may need to do some other processing here first. - - delete (it->second); - m_archiveFileMap.erase(it); -} - -void StdBIGFileSystem::closeAllArchiveFiles() { -} - -void StdBIGFileSystem::closeAllFiles() { -} - -Bool StdBIGFileSystem::loadBigFilesFromDirectory(AsciiString dir, AsciiString fileMask, Bool overwrite) { - - FilenameList filenameList; - TheLocalFileSystem->getFileListInDirectory(dir, "", fileMask, filenameList, TRUE); - - Bool actuallyAdded = FALSE; - FilenameListIter it = filenameList.begin(); - while (it != filenameList.end()) { -#if RTS_ZEROHOUR - // TheSuperHackers @bugfix bobtista 18/11/2025 Skip duplicate INIZH.big in Data\INI to prevent CRC mismatches. - // English, Chinese, and Korean SKUs shipped with two INIZH.big files (one in Run directory, one in Run\Data\INI). - // The DeleteFile cleanup doesn't work on EA App/Origin installs because the folder is not writable, so we skip loading it instead. - if (it->endsWithNoCase("Data\\INI\\INIZH.big") || it->endsWithNoCase("Data/INI/INIZH.big")) { - it++; - continue; - } -#endif - - ArchiveFile *archiveFile = openArchiveFile((*it).str()); - - if (archiveFile != nullptr) { - DEBUG_LOG(("StdBIGFileSystem::loadBigFilesFromDirectory - loading %s into the directory tree.", (*it).str())); - loadIntoDirectoryTree(archiveFile, overwrite); - m_archiveFileMap[(*it)] = archiveFile; - DEBUG_LOG(("StdBIGFileSystem::loadBigFilesFromDirectory - %s inserted into the archive file map.", (*it).str())); - actuallyAdded = TRUE; - } - - it++; - } - - return actuallyAdded; -} diff --git a/Core/GameEngineDevice/Source/StdDevice/Common/StdLocalFileSystem.cpp b/Core/GameEngineDevice/Source/StdDevice/Common/StdLocalFileSystem.cpp index d47e473f4d2..ee53db20549 100644 --- a/Core/GameEngineDevice/Source/StdDevice/Common/StdLocalFileSystem.cpp +++ b/Core/GameEngineDevice/Source/StdDevice/Common/StdLocalFileSystem.cpp @@ -304,10 +304,10 @@ Bool StdLocalFileSystem::getFileInfo(const AsciiString& filename, FileInfo *file // TODO: fix this to be win compatible (time since 1601) auto time = write_time.time_since_epoch().count(); + fileInfo->sizeHigh = file_size >> 32; + fileInfo->sizeLow = file_size & UINT32_MAX; fileInfo->timestampHigh = time >> 32; fileInfo->timestampLow = time & UINT32_MAX; - fileInfo->sizeHigh = file_size >> 32; - fileInfo->sizeLow = file_size & UINT32_MAX; return TRUE; } diff --git a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFile.cpp b/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFile.cpp deleted file mode 100644 index 3deee01fd82..00000000000 --- a/Core/GameEngineDevice/Source/Win32Device/Common/Win32BIGFile.cpp +++ /dev/null @@ -1,168 +0,0 @@ -/* -** Command & Conquer Generals Zero Hour(tm) -** Copyright 2025 Electronic Arts Inc. -** -** This program is free software: you can redistribute it and/or modify -** it under the terms of the GNU General Public License as published by -** the Free Software Foundation, either version 3 of the License, or -** (at your option) any later version. -** -** This program is distributed in the hope that it will be useful, -** but WITHOUT ANY WARRANTY; without even the implied warranty of -** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -** GNU General Public License for more details. -** -** You should have received a copy of the GNU General Public License -** along with this program. If not, see . -*/ - -//////////////////////////////////////////////////////////////////////////////// -// // -// (c) 2001-2003 Electronic Arts Inc. // -// // -//////////////////////////////////////////////////////////////////////////////// - -////// Win32BIGFile.cpp ///////////////////////// -// Bryan Cleveland, August 2002 -///////////////////////////////////////////////////// - -#include "Common/LocalFile.h" -#include "Common/LocalFileSystem.h" -#include "Common/RAMFile.h" -#include "Common/StreamingArchiveFile.h" -#include "Common/GameMemory.h" -#include "Common/PerfTimer.h" -#include "Win32Device/Common/Win32BIGFile.h" - -//============================================================================ -// Win32BIGFile::Win32BIGFile -//============================================================================ - -Win32BIGFile::Win32BIGFile(AsciiString name, AsciiString path) - : m_name(name) - , m_path(path) -{ - -} - -//============================================================================ -// Win32BIGFile::~Win32BIGFile -//============================================================================ - -Win32BIGFile::~Win32BIGFile() -{ - -} - -//============================================================================ -// Win32BIGFile::openFile -//============================================================================ - -File* Win32BIGFile::openFile( const Char *filename, Int access ) -{ - const ArchivedFileInfo *fileInfo = getArchivedFileInfo(AsciiString(filename)); - - if (fileInfo == nullptr) { - return nullptr; - } - - RAMFile *ramFile = nullptr; - - if (BitIsSet(access, File::STREAMING)) - ramFile = newInstance( StreamingArchiveFile ); - else - ramFile = newInstance( RAMFile ); - - ramFile->deleteOnClose(); - if (ramFile->openFromArchive(m_file, fileInfo->m_filename, fileInfo->m_offset, fileInfo->m_size) == FALSE) { - ramFile->close(); - ramFile = nullptr; - return nullptr; - } - - if ((access & File::WRITE) == 0) { - // requesting read only access. Just return the RAM file. - return ramFile; - } - - // whoever is opening this file wants write access, so copy the file to the local disk - // and return that file pointer. - - constexpr size_t bufferSize = 0; - File *localFile = TheLocalFileSystem->openFile(filename, access, bufferSize); - if (localFile != nullptr) { - ramFile->copyDataToFile(localFile); - } - - ramFile->close(); - ramFile = nullptr; - - return localFile; -} - -//============================================================================ -// Win32BIGFile::closeAllFiles -//============================================================================ - -void Win32BIGFile::closeAllFiles() -{ - -} - -//============================================================================ -// Win32BIGFile::getName -//============================================================================ - -AsciiString Win32BIGFile::getName() -{ - return m_name; -} - -//============================================================================ -// Win32BIGFile::getPath -//============================================================================ - -AsciiString Win32BIGFile::getPath() -{ - return m_path; -} - -//============================================================================ -// Win32BIGFile::setSearchPriority -//============================================================================ - -void Win32BIGFile::setSearchPriority( Int new_priority ) -{ - -} - -//============================================================================ -// Win32BIGFile::close -//============================================================================ - -void Win32BIGFile::close() -{ - -} - -//============================================================================ -// Win32BIGFile::getFileInfo -//============================================================================ - -Bool Win32BIGFile::getFileInfo(const AsciiString& filename, FileInfo *fileInfo) const -{ - const ArchivedFileInfo *tempFileInfo = getArchivedFileInfo(filename); - - if (tempFileInfo == nullptr) { - return FALSE; - } - - TheLocalFileSystem->getFileInfo(AsciiString(m_file->getName()), fileInfo); - - // fill in the size info. Since the size can't be bigger than a JUNK file, the high Int will always be 0. - fileInfo->sizeHigh = 0; - fileInfo->sizeLow = tempFileInfo->m_size; - - return TRUE; -} - diff --git a/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp b/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp index 43a6ca5cbdd..5ec6fd35680 100644 --- a/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp +++ b/Core/GameEngineDevice/Source/Win32Device/Common/Win32LocalFileSystem.cpp @@ -60,7 +60,7 @@ File * Win32LocalFileSystem::openFile(const Char *filename, Int access, size_t b AsciiString dirName; string.nextToken(&token, "\\/"); dirName = token; - while ((token.find('.') == nullptr) || (string.find('.') != nullptr)) { + while (!string.isEmpty()) { createDirectory(dirName); string.nextToken(&token, "\\/"); dirName.concat('\\'); @@ -194,10 +194,10 @@ Bool Win32LocalFileSystem::getFileInfo(const AsciiString& filename, FileInfo *fi return FALSE; } - fileInfo->timestampHigh = findData.ftLastWriteTime.dwHighDateTime; - fileInfo->timestampLow = findData.ftLastWriteTime.dwLowDateTime; fileInfo->sizeHigh = findData.nFileSizeHigh; fileInfo->sizeLow = findData.nFileSizeLow; + fileInfo->timestampHigh = findData.ftLastWriteTime.dwHighDateTime; + fileInfo->timestampLow = findData.ftLastWriteTime.dwLowDateTime; FindClose(findHandle); diff --git a/Core/Tools/Autorun/autorun.cpp b/Core/Tools/Autorun/autorun.cpp index 2e5a428c2b3..4f3618dcf2a 100644 --- a/Core/Tools/Autorun/autorun.cpp +++ b/Core/Tools/Autorun/autorun.cpp @@ -119,7 +119,7 @@ #include "GameClient/GameText.h" #include "Common/UnicodeString.h" #include "Win32Device/Common/Win32LocalFileSystem.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" +#include "Common/BIGFileSystem.h" #endif //----------------------------------------------------------------------------- @@ -449,7 +449,7 @@ int PASCAL WinMain( HINSTANCE hInstance, HINSTANCE hPrevInstance, LPTSTR lpszCmd // create singletons TheFileSystem->init(); TheLocalFileSystem = new Win32LocalFileSystem; - TheArchiveFileSystem = new Win32BIGFileSystem; + TheArchiveFileSystem = new BIGFileSystem; TheGameText = CreateGameTextInterface(); TheGameText->init(); diff --git a/Core/Tools/MapCacheBuilder/Source/WinMain.cpp b/Core/Tools/MapCacheBuilder/Source/WinMain.cpp index ab545a07892..fe7fcf1bc5d 100644 --- a/Core/Tools/MapCacheBuilder/Source/WinMain.cpp +++ b/Core/Tools/MapCacheBuilder/Source/WinMain.cpp @@ -51,8 +51,8 @@ #include "Common/ThingFactory.h" #include "Common/FileSystem.h" +#include "Common/BIGFileSystem.h" #include "Win32Device/Common/Win32LocalFileSystem.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" #include "Common/SubsystemInterface.h" #include "GameClient/MapUtil.h" #include "W3DDevice/Common/W3DModuleFactory.h" @@ -101,7 +101,6 @@ #include #include "Win32Device/GameClient/Win32Mouse.h" #include "Win32Device/Common/Win32LocalFileSystem.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" #include "WWLib/trim.h" @@ -249,7 +248,7 @@ Int APIENTRY WinMain( HINSTANCE hInstance, HINSTANCE hPrevInstance, TheFileSystem = new FileSystem; initSubsystem(TheLocalFileSystem, (LocalFileSystem*)new Win32LocalFileSystem); - initSubsystem(TheArchiveFileSystem, (ArchiveFileSystem*)new Win32BIGFileSystem); + initSubsystem(TheArchiveFileSystem, (ArchiveFileSystem*)new BIGFileSystem); INI ini; initSubsystem(TheWritableGlobalData, new GlobalData(), "Data\\INI\\Default\\GameData.ini", "Data\\INI\\GameData.ini"); initSubsystem(TheGameText, CreateGameTextInterface()); diff --git a/Generals/Code/GameEngineDevice/CMakeLists.txt b/Generals/Code/GameEngineDevice/CMakeLists.txt index dea3ff6c9ae..dbb41d0f7b4 100644 --- a/Generals/Code/GameEngineDevice/CMakeLists.txt +++ b/Generals/Code/GameEngineDevice/CMakeLists.txt @@ -70,8 +70,6 @@ set(GAMEENGINEDEVICE_SRC Include/W3DDevice/GameLogic/W3DGameLogic.h Include/W3DDevice/GameLogic/W3DGhostObject.h Include/W3DDevice/GameLogic/W3DTerrainLogic.h -# Include/Win32Device/Common/Win32BIGFile.h -# Include/Win32Device/Common/Win32BIGFileSystem.h Include/Win32Device/Common/Win32GameEngine.h # Include/Win32Device/Common/Win32LocalFile.h # Include/Win32Device/Common/Win32LocalFileSystem.h @@ -168,8 +166,6 @@ set(GAMEENGINEDEVICE_SRC Source/W3DDevice/GameLogic/W3DGameLogic.cpp Source/W3DDevice/GameLogic/W3DGhostObject.cpp Source/W3DDevice/GameLogic/W3DTerrainLogic.cpp -# Source/Win32Device/Common/Win32BIGFile.cpp -# Source/Win32Device/Common/Win32BIGFileSystem.cpp Source/Win32Device/Common/Win32GameEngine.cpp # Source/Win32Device/Common/Win32LocalFile.cpp # Source/Win32Device/Common/Win32LocalFileSystem.cpp diff --git a/Generals/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h b/Generals/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h index b17369dc844..01f5b0af14f 100644 --- a/Generals/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h +++ b/Generals/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h @@ -37,7 +37,7 @@ #include "GameLogic/GameLogic.h" #include "GameNetwork/NetworkInterface.h" #include "MilesAudioDevice/MilesAudioManager.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" +#include "Common/BIGFileSystem.h" #include "Win32Device/Common/Win32LocalFileSystem.h" #include "W3DDevice/Common/W3DModuleFactory.h" #include "W3DDevice/GameLogic/W3DGameLogic.h" @@ -93,7 +93,7 @@ inline ModuleFactory *Win32GameEngine::createModuleFactory() { return NEW W3DMod inline ThingFactory *Win32GameEngine::createThingFactory() { return NEW W3DThingFactory; } inline FunctionLexicon *Win32GameEngine::createFunctionLexicon() { return NEW W3DFunctionLexicon; } inline LocalFileSystem *Win32GameEngine::createLocalFileSystem() { return NEW Win32LocalFileSystem; } -inline ArchiveFileSystem *Win32GameEngine::createArchiveFileSystem() { return NEW Win32BIGFileSystem; } +inline ArchiveFileSystem *Win32GameEngine::createArchiveFileSystem() { return NEW BIGFileSystem; } inline ParticleSystemManager* Win32GameEngine::createParticleSystemManager(Bool dummy) { if (dummy) diff --git a/Generals/Code/Tools/GUIEdit/Source/GUIEdit.cpp b/Generals/Code/Tools/GUIEdit/Source/GUIEdit.cpp index 4d18b320cd8..d5c928ff52a 100644 --- a/Generals/Code/Tools/GUIEdit/Source/GUIEdit.cpp +++ b/Generals/Code/Tools/GUIEdit/Source/GUIEdit.cpp @@ -85,7 +85,7 @@ #include "Win32Device/GameClient/Win32Mouse.h" #include "Win32Device/GameClient/Win32DIKeyboard.h" #include "Win32Device/Common/Win32LocalFileSystem.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" +#include "Common/BIGFileSystem.h" #include "resource.h" #include "WinMain.h" @@ -485,7 +485,7 @@ void GUIEdit::init() // create file system TheFileSystem = new FileSystem; TheLocalFileSystem = new Win32LocalFileSystem; - TheArchiveFileSystem = new Win32BIGFileSystem; + TheArchiveFileSystem = new BIGFileSystem; TheFileSystem->init(); //--------------------------------------------------------------------------- diff --git a/Generals/Code/Tools/WorldBuilder/src/WorldBuilder.cpp b/Generals/Code/Tools/WorldBuilder/src/WorldBuilder.cpp index b3287235361..9c5421c320d 100644 --- a/Generals/Code/Tools/WorldBuilder/src/WorldBuilder.cpp +++ b/Generals/Code/Tools/WorldBuilder/src/WorldBuilder.cpp @@ -79,7 +79,7 @@ #include #include "Win32Device/GameClient/Win32Mouse.h" #include "Win32Device/Common/Win32LocalFileSystem.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" +#include "Common/BIGFileSystem.h" static SubsystemInterfaceList TheSubsystemListRecord; @@ -316,7 +316,7 @@ BOOL CWorldBuilderApp::InitInstance() TheFileSystem = new FileSystem; initSubsystem(TheLocalFileSystem, (LocalFileSystem*)new Win32LocalFileSystem); - initSubsystem(TheArchiveFileSystem, (ArchiveFileSystem*)new Win32BIGFileSystem); + initSubsystem(TheArchiveFileSystem, (ArchiveFileSystem*)new BIGFileSystem); // Just for kicks, get the HINSTANCE that WOL would need // if we were going to use it, which we aren't. diff --git a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.h b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.h index 0bcc1a2c3e3..8ac6d4fb329 100644 --- a/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.h +++ b/GeneralsMD/Code/GameEngine/Include/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.h @@ -10,6 +10,13 @@ enum EHTTPVersion HTTP_VERSION_3_0 }; +// One layer of data_packs in settings.json: BIG paths in priority order, the first one wins +struct GenOnlineDataPackLayer +{ + bool m_enabled = true; + std::vector m_includePaths; +}; + class GenOnlineSettings { public: @@ -70,7 +77,10 @@ class GenOnlineSettings Load(); } - bool DataPacks_UseCommunityPatch() const { return m_DataPacks_UseCommunityPatch; } + const GenOnlineDataPackLayer& DataPacks_User() const { return m_DataPacks_User; } + const GenOnlineDataPackLayer& DataPacks_GeneralsOnline() const { return m_DataPacks_GeneralsOnline; } + const GenOnlineDataPackLayer& DataPacks_CommunityPatch() const { return m_DataPacks_CommunityPatch; } + const std::vector& DataPacks_UserExcludePaths() const { return m_DataPacks_UserExcludePaths; } bool Network_UseAlternativeEndpoint() const { return m_Network_UseAlternativeEndpoint; } EHTTPVersion Network_GetHTTPVersion() const { return m_Network_HTTPVersion; } @@ -139,7 +149,11 @@ class GenOnlineSettings bool m_Social_Notification_PlayerSendsRequest_Menus = true; bool m_Social_Notification_PlayerSendsRequest_Gameplay = true; - bool m_DataPacks_UseCommunityPatch = true; + // BIG archives above the retail ones. The layers load in this order, highest priority first. A missing or invalid field keeps its default. + GenOnlineDataPackLayer m_DataPacks_User; + std::vector m_DataPacks_UserExcludePaths; // loose files of the game folder, BIGs of the game folders and BIGs of the user layer that are left out + GenOnlineDataPackLayer m_DataPacks_GeneralsOnline = { true, { "user:GeneralsOnlineGameData/GeneralsOnline_Core.big" } }; + GenOnlineDataPackLayer m_DataPacks_CommunityPatch = { true, { "user:GeneralsOnlineGameData/500_900_CommunityPatch_CoreINI.big" } }; std::string m_Plugins_Anticheat = std::string(); diff --git a/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp b/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp index 0c497745de0..63e64f71f76 100644 --- a/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp +++ b/GeneralsMD/Code/GameEngine/Source/Common/GameEngine.cpp @@ -504,6 +504,10 @@ void GameEngine::init() #endif///////////////////////////////////////////////////////////////////////////////////////////// + // NGMP_CHANGE: Init our settings before the archives are mounted, the data_packs layers decide which ones. Needs TheGlobalData for the user data path. + NGMP_OnlineServicesManager::Settings.Initialize(); + TheFileSystem->setExcludedPaths(NGMP_OnlineServicesManager::Settings.DataPacks_UserExcludePaths()); + initSubsystem(TheArchiveFileSystem, "TheArchiveFileSystem", createArchiveFileSystem(), nullptr); // this MUST come after TheLocalFileSystem creation #ifdef DUMP_PERF_STATS/////////////////////////////////////////////////////////////////////////// @@ -538,9 +542,6 @@ void GameEngine::init() // special-case: parse command-line parameters after loading global data CommandLine::parseCommandLineForEngineInit(); - // NGMP_CHANGE: Init our settings before loadMods, which reads DataPacks_UseCommunityPatch. Needs TheGlobalData for the user data path. - NGMP_OnlineServicesManager::Settings.Initialize(); - TheArchiveFileSystem->loadMods(); // doesn't require resets so just create a single instance here. diff --git a/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GameWindowManagerScript.cpp b/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GameWindowManagerScript.cpp index d588e90cc31..d1d2da5bea0 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GameWindowManagerScript.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameClient/GUI/GameWindowManagerScript.cpp @@ -2726,26 +2726,6 @@ GameWindow *GameWindowManager::winCreateFromScript( AsciiString filenameString, else strlcpy(filepath, filename, ARRAY_SIZE(filepath)); -#if defined(GENERALS_ONLINE) - // check Generals Online first - char gofilepath[_MAX_PATH] = "GeneralsOnlineGameData\\"; - if (strchr(filename, '\\') == NULL) - sprintf(gofilepath, "GeneralsOnlineGameData\\%s", filename); - else - strcpy(gofilepath, filename); - - inFile = TheFileSystem->openFile(gofilepath, File::READ); - if (inFile == NULL) - { - // fall back to game archive - inFile = TheFileSystem->openFile(filepath, File::READ); - if (inFile == NULL) - { - DEBUG_LOG(("WinCreateFromScript: Cannot access file '%s'.\n", filename)); - return NULL; - } - } -#else // Open the input file inFile = TheFileSystem->openFile(filepath, File::READ); if (inFile == nullptr) @@ -2753,7 +2733,6 @@ GameWindow *GameWindowManager::winCreateFromScript( AsciiString filenameString, DEBUG_LOG(( "WinCreateFromScript: Cannot access file '%s'.", filename )); return nullptr; } -#endif // read into memory inFile=inFile->convertToRAMFile(); diff --git a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.cpp b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.cpp index 347c65de981..b8e3ca9bfbc 100644 --- a/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.cpp +++ b/GeneralsMD/Code/GameEngine/Source/GameNetwork/GeneralsOnline/GeneralsOnline_Settings.cpp @@ -1,4 +1,5 @@ #include "GameNetwork/GeneralsOnline/GeneralsOnline_Settings.h" +#include "Common/ArchiveFileSystem.h" #include "../json.hpp" #include "../OnlineServices_LobbyInterface.h" #include "../OnlineServices_Init.h" @@ -37,7 +38,13 @@ #define SETTINGS_KEY_PLUGINS_ANTICHEAT "anticheat" #define SETTINGS_KEY_DATAPACKS "data_packs" -#define SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH "use_community_data_patch" +#define SETTINGS_KEY_DATAPACKS_USER "user" +#define SETTINGS_KEY_DATAPACKS_GENERALS_ONLINE "generals_online" +#define SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH "community_patch" +#define SETTINGS_KEY_DATAPACKS_ENABLED "enabled" +#define SETTINGS_KEY_DATAPACKS_INCLUDE_PATHS "include_paths" +#define SETTINGS_KEY_DATAPACKS_EXCLUDE_PATHS "exclude_paths" +#define SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH_LEGACY "use_community_data_patch" // before the layers, read once and dropped on save #define SETTINGS_FILENAME_LEGACY "GeneralsOnline_settings.json" #define SETTINGS_FILENAME "settings.json" @@ -94,6 +101,89 @@ std::string GenOnlineSettings::GetAnticheatPlugin() const return m_Plugins_Anticheat; } +// Reads a list of paths of data_packs, an entry that is not valid is dropped. Returns FALSE and keeps the list when the field is missing or not a list. +static bool LoadDataPackPaths(const nlohmann::json& jsonLayer, const char* layerName, const char* key, bool excludes, std::vector& paths) +{ + if (!jsonLayer.contains(key)) + { + return false; + } + + if (!jsonLayer[key].is_array()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Data pack layer %s: %s is not a list, using the default", layerName, key); + return false; + } + + paths.clear(); + for (const auto& path : jsonLayer[key]) + { + AsciiString reason; + bool bValid = false; + if (path.is_string()) + { + bValid = excludes ? PathExcludeList::parse(path.get(), nullptr, &reason) : ArchiveFileSystem::parseMountEntry(AsciiString(path.get().c_str()), nullptr, &reason); + } + + if (bValid) + { + paths.push_back(path.get()); + } + else + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Data pack layer %s: %s entry dropped from the settings (%s)", layerName, key, path.is_string() ? reason.str() : "not a string"); + } + } + return true; +} + +// Reads one layer of data_packs. A missing or invalid field keeps what the layer holds, its default. excludePaths is only read for the user layer. +static void LoadDataPackLayer(const nlohmann::json& dataPacks, const char* name, GenOnlineDataPackLayer& layer, std::vector* excludePaths) +{ + if (!dataPacks.contains(name)) + { + return; + } + + const nlohmann::json& jsonLayer = dataPacks[name]; + if (!jsonLayer.is_object()) + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Data pack layer %s in the settings is not an object, using the default", name); + return; + } + + if (jsonLayer.contains(SETTINGS_KEY_DATAPACKS_ENABLED)) + { + if (jsonLayer[SETTINGS_KEY_DATAPACKS_ENABLED].is_boolean()) + { + layer.m_enabled = jsonLayer[SETTINGS_KEY_DATAPACKS_ENABLED].get(); + } + else + { + NetworkLog(ELogVerbosity::LOG_RELEASE, "Data pack layer %s: enabled is not a boolean, using the default", name); + } + } + + LoadDataPackPaths(jsonLayer, name, SETTINGS_KEY_DATAPACKS_INCLUDE_PATHS, false, layer.m_includePaths); + + if (excludePaths != nullptr) + { + LoadDataPackPaths(jsonLayer, name, SETTINGS_KEY_DATAPACKS_EXCLUDE_PATHS, true, *excludePaths); + } +} + +static nlohmann::json DataPackLayerToJson(const GenOnlineDataPackLayer& layer, const std::vector* excludePaths) +{ + nlohmann::json jsonLayer = nlohmann::json::object(); + jsonLayer[SETTINGS_KEY_DATAPACKS_ENABLED] = layer.m_enabled; + jsonLayer[SETTINGS_KEY_DATAPACKS_INCLUDE_PATHS] = layer.m_includePaths; + if (excludePaths != nullptr) + { + jsonLayer[SETTINGS_KEY_DATAPACKS_EXCLUDE_PATHS] = *excludePaths; + } + return jsonLayer; +} + void GenOnlineSettings::Load(void) { char GameDir[MAX_PATH + 1] = {}; @@ -317,11 +407,18 @@ void GenOnlineSettings::Load(void) if (jsonSettings.contains(SETTINGS_KEY_DATAPACKS)) { - auto datapackSettings = jsonSettings[SETTINGS_KEY_DATAPACKS]; + const nlohmann::json& datapackSettings = jsonSettings[SETTINGS_KEY_DATAPACKS]; + + LoadDataPackLayer(datapackSettings, SETTINGS_KEY_DATAPACKS_USER, m_DataPacks_User, &m_DataPacks_UserExcludePaths); + LoadDataPackLayer(datapackSettings, SETTINGS_KEY_DATAPACKS_GENERALS_ONLINE, m_DataPacks_GeneralsOnline, nullptr); + LoadDataPackLayer(datapackSettings, SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH, m_DataPacks_CommunityPatch, nullptr); - if (datapackSettings.contains(SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH)) + // a file from before the layers: the old community patch switch becomes community_patch.enabled + if (!datapackSettings.contains(SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH) + && datapackSettings.contains(SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH_LEGACY) + && datapackSettings[SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH_LEGACY].is_boolean()) { - m_DataPacks_UseCommunityPatch = datapackSettings[SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH]; + m_DataPacks_CommunityPatch.m_enabled = datapackSettings[SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH_LEGACY].get(); } } } @@ -443,7 +540,9 @@ void GenOnlineSettings::Save() { SETTINGS_KEY_DATAPACKS, { - {SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH, m_DataPacks_UseCommunityPatch} + {SETTINGS_KEY_DATAPACKS_USER, DataPackLayerToJson(m_DataPacks_User, &m_DataPacks_UserExcludePaths)}, + {SETTINGS_KEY_DATAPACKS_GENERALS_ONLINE, DataPackLayerToJson(m_DataPacks_GeneralsOnline, nullptr)}, + {SETTINGS_KEY_DATAPACKS_COMMUNITY_PATCH, DataPackLayerToJson(m_DataPacks_CommunityPatch, nullptr)} } } }; diff --git a/GeneralsMD/Code/GameEngineDevice/CMakeLists.txt b/GeneralsMD/Code/GameEngineDevice/CMakeLists.txt index 56408e821bd..d251e04bb38 100644 --- a/GeneralsMD/Code/GameEngineDevice/CMakeLists.txt +++ b/GeneralsMD/Code/GameEngineDevice/CMakeLists.txt @@ -81,8 +81,6 @@ set(GAMEENGINEDEVICE_SRC Include/W3DDevice/GameLogic/W3DGameLogic.h Include/W3DDevice/GameLogic/W3DGhostObject.h Include/W3DDevice/GameLogic/W3DTerrainLogic.h -# Include/Win32Device/Common/Win32BIGFile.h -# Include/Win32Device/Common/Win32BIGFileSystem.h Include/Win32Device/Common/Win32GameEngine.h # Include/Win32Device/Common/Win32LocalFile.h # Include/Win32Device/Common/Win32LocalFileSystem.h @@ -181,8 +179,6 @@ set(GAMEENGINEDEVICE_SRC Source/W3DDevice/GameLogic/W3DGameLogic.cpp Source/W3DDevice/GameLogic/W3DGhostObject.cpp Source/W3DDevice/GameLogic/W3DTerrainLogic.cpp -# Source/Win32Device/Common/Win32BIGFile.cpp -# Source/Win32Device/Common/Win32BIGFileSystem.cpp Source/Win32Device/Common/Win32GameEngine.cpp # Source/Win32Device/Common/Win32LocalFile.cpp # Source/Win32Device/Common/Win32LocalFileSystem.cpp diff --git a/GeneralsMD/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h b/GeneralsMD/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h index 2710cdf4555..e27eb8504c9 100644 --- a/GeneralsMD/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h +++ b/GeneralsMD/Code/GameEngineDevice/Include/Win32Device/Common/Win32GameEngine.h @@ -32,12 +32,12 @@ #pragma once +#include "Common/BIGFileSystem.h" #include "Common/GameEngine.h" #include "GameClient/ParticleSys.h" #include "GameLogic/GameLogic.h" #include "GameNetwork/NetworkInterface.h" #include "MilesAudioDevice/MilesAudioManager.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" #include "Win32Device/Common/Win32LocalFileSystem.h" #include "W3DDevice/Common/W3DModuleFactory.h" #include "W3DDevice/GameLogic/W3DGameLogic.h" @@ -93,7 +93,7 @@ inline ModuleFactory *Win32GameEngine::createModuleFactory() { return NEW W3DMod inline ThingFactory *Win32GameEngine::createThingFactory() { return NEW W3DThingFactory; } inline FunctionLexicon *Win32GameEngine::createFunctionLexicon() { return NEW W3DFunctionLexicon; } inline LocalFileSystem *Win32GameEngine::createLocalFileSystem() { return NEW Win32LocalFileSystem; } -inline ArchiveFileSystem *Win32GameEngine::createArchiveFileSystem() { return NEW Win32BIGFileSystem; } +inline ArchiveFileSystem *Win32GameEngine::createArchiveFileSystem() { return NEW BIGFileSystem; } inline ParticleSystemManager* Win32GameEngine::createParticleSystemManager(Bool dummy) { if (dummy) diff --git a/GeneralsMD/Code/Tools/GUIEdit/Source/GUIEdit.cpp b/GeneralsMD/Code/Tools/GUIEdit/Source/GUIEdit.cpp index 2cbc8ec3ffb..6ecfad34952 100644 --- a/GeneralsMD/Code/Tools/GUIEdit/Source/GUIEdit.cpp +++ b/GeneralsMD/Code/Tools/GUIEdit/Source/GUIEdit.cpp @@ -85,7 +85,7 @@ #include "Win32Device/GameClient/Win32Mouse.h" #include "Win32Device/GameClient/Win32DIKeyboard.h" #include "Win32Device/Common/Win32LocalFileSystem.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" +#include "Common/BIGFileSystem.h" #include "resource.h" #include "WinMain.h" @@ -485,7 +485,7 @@ void GUIEdit::init() // create file system TheFileSystem = new FileSystem; TheLocalFileSystem = new Win32LocalFileSystem; - TheArchiveFileSystem = new Win32BIGFileSystem; + TheArchiveFileSystem = new BIGFileSystem; TheFileSystem->init(); //--------------------------------------------------------------------------- diff --git a/GeneralsMD/Code/Tools/WorldBuilder/src/WorldBuilder.cpp b/GeneralsMD/Code/Tools/WorldBuilder/src/WorldBuilder.cpp index d49589ee0c8..65d2fb82b8f 100644 --- a/GeneralsMD/Code/Tools/WorldBuilder/src/WorldBuilder.cpp +++ b/GeneralsMD/Code/Tools/WorldBuilder/src/WorldBuilder.cpp @@ -79,7 +79,7 @@ #include #include "Win32Device/GameClient/Win32Mouse.h" #include "Win32Device/Common/Win32LocalFileSystem.h" -#include "Win32Device/Common/Win32BIGFileSystem.h" +#include "Common/BIGFileSystem.h" static SubsystemInterfaceList TheSubsystemListRecord; @@ -326,7 +326,7 @@ BOOL CWorldBuilderApp::InitInstance() TheFileSystem = new FileSystem; initSubsystem(TheLocalFileSystem, (LocalFileSystem*)new Win32LocalFileSystem); - initSubsystem(TheArchiveFileSystem, (ArchiveFileSystem*)new Win32BIGFileSystem); + initSubsystem(TheArchiveFileSystem, (ArchiveFileSystem*)new BIGFileSystem); // Just for kicks, get the HINSTANCE that WOL would need // if we were going to use it, which we aren't. diff --git a/scripts/pack_go_data.py b/scripts/pack_go_data.py new file mode 100644 index 00000000000..a084b052273 --- /dev/null +++ b/scripts/pack_go_data.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +r"""Pack GeneralsOnline/GameData into the Generals Online fix pack BIG. + +Usage: pack_go_data.py [output.big] + +The folder layout inside GameData is the layout inside the BIG (Window\..., Data\INI\..., Art\Textures\...). +The updater ships the result as \GeneralsOnlineGameData\GeneralsOnline_Core.big. + +The client mounts it through the generals_online layer of data_packs in settings.json. From the top, the lookup is: loose files in the +game folder, -mod and -modDir, the BIGs in the game folders that are not retail, then the layers user, generals_online and community_patch, +then the retail archives. Every layer has enabled and include_paths, which are mounted in list order (the first one wins). An entry is +game:, user: or an absolute path: a BIG, a folder of BIGs or a *.big pattern. + + "data_packs": { + "user": { "enabled": true, "include_paths": [], "exclude_paths": [] }, + "generals_online": { "enabled": true, "include_paths": [ "user:GeneralsOnlineGameData/GeneralsOnline_Core.big" ] }, + "community_patch": { "enabled": true, "include_paths": [ "user:GeneralsOnlineGameData/500_900_CommunityPatch_CoreINI.big" ] } + } + +user.exclude_paths leaves things out: loose files in the game folder (game:Data/INI/*), the BIGs in the game folders that are not retail +(game:340_ControlBarPro*.big, game:ZH_Generals/*.big) and the include_paths of the user layer itself. * matches any text including folders, +a trailing slash is a folder. Retail archives are never excluded. user.enabled false only turns off user.include_paths. + +The old data_packs.use_community_data_patch switch is read as community_patch.enabled when community_patch is missing. +-disableCommunityDataPatch skips the community_patch layer for one launch. +""" +import os +import struct +import sys + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "GeneralsOnline", "GameData") +OUT = sys.argv[1] if len(sys.argv) > 1 else "GeneralsOnline_Core.big" + + +def main(): + files = [] + for dirpath, _, names in os.walk(ROOT): + for name in names: + full = os.path.join(dirpath, name) + rel = os.path.relpath(full, ROOT).replace("/", "\\") + files.append((rel, full)) + files.sort(key=lambda f: f[0].lower()) + if not files: + sys.exit("nothing to pack in " + ROOT) + + names = [rel.encode("ascii") + b"\0" for rel, _ in files] + header = 16 + sum(8 + len(n) for n in names) + table = b"" + body = b"" + offset = header + for (rel, full), n in zip(files, names): + with open(full, "rb") as f: + data = f.read() + table += struct.pack(">II", offset, len(data)) + n + body += data + offset += len(data) + # BIGF: magic, total size (little endian), file count and data start (big endian) + with open(OUT, "wb") as f: + f.write(b"BIGF" + struct.pack("II", len(files), header) + table + body) + print("packed %d files into %s" % (len(files), OUT)) + + +main()