quick/netradiant-custom
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

* sort pk3s, so content of later (zzz) overrides earlier, like in radiant and engine

Browse Source 
* fix strong performance penalty with large amount of files in pk3s
store pak file path once per pk3, not per each file inside
This commit is contained in:
Garux 2021-08-13 03:20:07 +03:00
parent 4beae3d362
commit bb1931b745
3 changed files with 90 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
libs/string/string.h
View File

@ -129,6 +129,27 @@ inline bool string_greater_nocase( const char* string, const char* other ){
return string_compare_nocase( string, other ) > 0; return string_compare_nocase( string, other ) > 0;
} }
/*!
This behaves identically to stricmp(a,b), except that ASCII chars
[\]^`_ come AFTER alphabet chars instead of before. This is because
it converts all alphabet chars to uppercase before comparison,
while stricmp converts them to lowercase.
*/
inline int string_compare_nocase_upper( const char* a, const char* b ){
for (;; )
{
const int c1 = std::toupper( *a++ );
const int c2 = std::toupper( *b++ );
if ( c1 < c2 )
return -1; // a < b
if ( c1 > c2 )
return 1; // a > b
if ( c1 == 0 )
return 0; // a == b
}
}
/// \brief Returns the number of non-null characters in \p string. /// \brief Returns the number of non-null characters in \p string.
/// O(n) /// O(n)
inline std::size_t string_length( const char* string ){ inline std::size_t string_length( const char* string ){

33
plugins/vfspk3/vfs.cpp
View File

@ -92,7 +92,7 @@ static char g_strDirs[VFS_MAXDIRS][PATH_MAX + 1];
static int g_numDirs; static int g_numDirs;
static char g_strForbiddenDirs[VFS_MAXDIRS][PATH_MAX + 1]; static char g_strForbiddenDirs[VFS_MAXDIRS][PATH_MAX + 1];
static int g_numForbiddenDirs = 0; static int g_numForbiddenDirs = 0;
static bool g_bUsePak = true; static constexpr bool g_bUsePak = true;
ModuleObservers g_observers; ModuleObservers g_observers;
@ -228,37 +228,6 @@ static GSList* GetListInternal( const char *refdir, const char *ext, bool direct
return files; return files;
} }
inline int ascii_to_upper( int c ){
if ( c >= 'a' && c <= 'z' ) {
return c - ( 'a' - 'A' );
}
return c;
}
/*!
This behaves identically to stricmp(a,b), except that ASCII chars
[\]^`_ come AFTER alphabet chars instead of before. This is because
it converts all alphabet chars to uppercase before comparison,
while stricmp converts them to lowercase.
*/
static int string_compare_nocase_upper( const char* a, const char* b ){
for (;; )
{
int c1 = ascii_to_upper( *a++ );
int c2 = ascii_to_upper( *b++ );
if ( c1 < c2 ) {
return -1; // a < b
}
if ( c1 > c2 ) {
return 1; // a > b
}
if ( c1 == 0 ) {
return 0; // a == b
}
}
}
// Arnout: note - sort pakfiles in reverse order. This ensures that // Arnout: note - sort pakfiles in reverse order. This ensures that
// later pakfiles override earlier ones. This because the vfs module // later pakfiles override earlier ones. This because the vfs module
// returns a filehandle to the first file it can find (while it should // returns a filehandle to the first file it can find (while it should

139
tools/quake3/common/vfs.cpp
View File

@ -54,26 +54,36 @@
#include "stream/stringstream.h" #include "stream/stringstream.h"
#include "stream/textstream.h" #include "stream/textstream.h"
#include <forward_list>
struct VFS_PAK
{
unzFile zipfile;
const CopiedString unzFilePath;
VFS_PAK( unzFile zipfile, const char *unzFilePath ) : zipfile( zipfile ), unzFilePath( unzFilePath ) {};
~VFS_PAK(){
unzClose( zipfile );
}
};
struct VFS_PAKFILE struct VFS_PAKFILE
{ {
char* unzFilePath; const CopiedString name;
char* name; const unz_s zipinfo;
unz_s zipinfo; VFS_PAK& pak;
unzFile zipfile; const guint32 size;
guint32 size;
}; };
// ============================================================================= // =============================================================================
// Global variables // Global variables
static GSList* g_unzFiles; static std::forward_list<VFS_PAK> g_paks;
static GSList* g_pakFiles; static std::forward_list<VFS_PAKFILE> g_pakFiles;
static char g_strDirs[VFS_MAXDIRS][PATH_MAX + 1]; static char g_strDirs[VFS_MAXDIRS][PATH_MAX + 1];
static int g_numDirs; static int g_numDirs;
char g_strForbiddenDirs[VFS_MAXDIRS][PATH_MAX + 1]; char g_strForbiddenDirs[VFS_MAXDIRS][PATH_MAX + 1];
int g_numForbiddenDirs = 0; int g_numForbiddenDirs = 0;
static gboolean g_bUsePak = TRUE; static constexpr bool g_bUsePak = true;
char g_strLoadedFileLocation[1024]; char g_strLoadedFileLocation[1024];
// ============================================================================= // =============================================================================
@ -93,7 +103,7 @@ static void vfsInitPakFile( const char *filename ){
return; return;
} }
g_unzFiles = g_slist_append( g_unzFiles, uf ); VFS_PAK& pak = g_paks.emplace_front( uf, filename );
err = unzGetGlobalInfo( uf,&gi ); err = unzGetGlobalInfo( uf,&gi );
if ( err != UNZ_OK ) { if ( err != UNZ_OK ) {
@ -101,8 +111,6 @@ static void vfsInitPakFile( const char *filename ){
} }
unzGoToFirstFile( uf ); unzGoToFirstFile( uf );
char* unzFilePath = strdup( filename );
for ( i = 0; i < gi.number_entry; i++ ) for ( i = 0; i < gi.number_entry; i++ )
{ {
char filename_inzip[NAME_MAX]; char filename_inzip[NAME_MAX];
@ -113,17 +121,15 @@ static void vfsInitPakFile( const char *filename ){
break; break;
} }
VFS_PAKFILE* file = safe_malloc( sizeof( VFS_PAKFILE ) );
g_pakFiles = g_slist_append( g_pakFiles, file );
FixDOSName( filename_inzip ); FixDOSName( filename_inzip );
strLower( filename_inzip ); strLower( filename_inzip );
file->name = strdup( filename_inzip ); g_pakFiles.emplace_front( VFS_PAKFILE{
file->size = file_info.uncompressed_size; filename_inzip,
file->zipfile = uf; *(unz_s*)uf,
file->unzFilePath = unzFilePath; pak,
memcpy( &file->zipinfo, uf, sizeof( unz_s ) ); file_info.uncompressed_size
} );
if ( ( i + 1 ) < gi.number_entry ) { if ( ( i + 1 ) < gi.number_entry ) {
err = unzGoToNextFile( uf ); err = unzGoToNextFile( uf );
@ -139,7 +145,6 @@ static void vfsInitPakFile( const char *filename ){
// reads all pak files from a dir // reads all pak files from a dir
void vfsInitDirectory( const char *path ){ void vfsInitDirectory( const char *path ){
char filename[PATH_MAX];
GDir *dir; GDir *dir;
int j; int j;
@ -171,6 +176,7 @@ void vfsInitDirectory( const char *path ){
dir = g_dir_open( path, 0, NULL ); dir = g_dir_open( path, 0, NULL );
if ( dir != NULL ) { if ( dir != NULL ) {
std::vector<StringOutputStream> paks;
const char* name; const char* name;
while ( ( name = g_dir_read_name( dir ) ) ) while ( ( name = g_dir_read_name( dir ) ) )
{ {
@ -187,8 +193,7 @@ void vfsInitDirectory( const char *path ){
const char *ext = path_get_filename_base_end( name ); const char *ext = path_get_filename_base_end( name );
if ( striEqual( ext, ".pk3" ) ) { if ( striEqual( ext, ".pk3" ) ) {
sprintf( filename, "%s/%s", path, name ); paks.push_back( StringOutputStream( 256 )( path, '/', name ) );
vfsInitPakFile( filename );
} }
else if ( striEqual( ext, ".pk3dir" ) ) { else if ( striEqual( ext, ".pk3dir" ) ) {
if ( g_numDirs == VFS_MAXDIRS ) { if ( g_numDirs == VFS_MAXDIRS ) {
@ -201,6 +206,17 @@ void vfsInitDirectory( const char *path ){
} }
} }
g_dir_close( dir ); g_dir_close( dir );
// sort paks in ascending order
// pakFiles are then prepended to the list, reversing the order
// thus later (zzz) pak content have priority over earlier, just like in engine
std::sort( paks.begin(), paks.end(),
[]( const char* a, const char* b ){
return string_compare_nocase_upper( a, b ) < 0;
} );
for( const char* pak : paks ){
vfsInitPakFile( pak );
}
} }
} }
} }
@ -231,12 +247,12 @@ std::vector<CopiedString> vfsListShaderFiles( const char *shaderPath ){
} }
} }
/* search in packs */ /* search in packs */
for ( GSList *lst = g_pakFiles; lst != NULL; lst = g_slist_next( lst ) ) for ( const VFS_PAKFILE& file : g_pakFiles )
{ {
VFS_PAKFILE* file = (VFS_PAKFILE*)lst->data; const char *name = file.name.c_str();
if ( striEqual( path_get_filename_base_end( file->name ), ".shader" ) if ( striEqual( path_get_filename_base_end( name ), ".shader" )
&& strniEqual( file->name, shaderPath, path_get_last_separator( file->name ) - file->name ) ) { && strniEqual( name, shaderPath, path_get_last_separator( name ) - name ) ) {
insert( path_get_filename_start( file->name ) ); insert( path_get_filename_start( name ) );
} }
} }
@ -245,37 +261,22 @@ std::vector<CopiedString> vfsListShaderFiles( const char *shaderPath ){
// frees all memory that we allocated // frees all memory that we allocated
void vfsShutdown(){ void vfsShutdown(){
while ( g_unzFiles ) g_paks.clear();
{ g_pakFiles.clear();
unzClose( (unzFile)g_unzFiles->data );
g_unzFiles = g_slist_remove( g_unzFiles, g_unzFiles->data );
}
while ( g_pakFiles )
{
VFS_PAKFILE* file = (VFS_PAKFILE*)g_pakFiles->data;
free( file->unzFilePath );
free( file->name );
free( file );
g_pakFiles = g_slist_remove( g_pakFiles, file );
}
} }
// return the number of files that match // return the number of files that match
int vfsGetFileCount( const char *filename ){ int vfsGetFileCount( const char *filename ){
int i, count = 0; int i, count = 0;
char fixed[NAME_MAX], tmp[NAME_MAX]; char fixed[NAME_MAX], tmp[NAME_MAX];
GSList *lst;
strcpy( fixed, filename ); strcpy( fixed, filename );
FixDOSName( fixed ); FixDOSName( fixed );
strLower( fixed ); strLower( fixed );
for ( lst = g_pakFiles; lst != NULL; lst = g_slist_next( lst ) ) for ( const VFS_PAKFILE& file : g_pakFiles )
{ {
VFS_PAKFILE* file = (VFS_PAKFILE*)lst->data; if ( strEqual( file.name.c_str(), fixed ) ) {
if ( strEqual( file->name, fixed ) ) {
count++; count++;
} }
} }
@ -296,7 +297,6 @@ int vfsGetFileCount( const char *filename ){
int vfsLoadFile( const char *filename, void **bufferptr, int index ){ int vfsLoadFile( const char *filename, void **bufferptr, int index ){
int i, count = 0; int i, count = 0;
char tmp[NAME_MAX], fixed[NAME_MAX]; char tmp[NAME_MAX], fixed[NAME_MAX];
GSList *lst;
// filename is a full path // filename is a full path
if ( index == -1 ) { if ( index == -1 ) {
@ -365,34 +365,33 @@ int vfsLoadFile( const char *filename, void **bufferptr, int index ){
} }
} }
for ( lst = g_pakFiles; lst != NULL; lst = g_slist_next( lst ) ) for ( const VFS_PAKFILE& file : g_pakFiles )
{ {
VFS_PAKFILE* file = (VFS_PAKFILE*)lst->data; if ( !strEqual( file.name.c_str(), fixed ) ) {
if ( !strEqual( file->name, fixed ) ) {
continue; continue;
} }
if ( count == index ) { if ( count == index ) {
snprintf( g_strLoadedFileLocation, sizeof( g_strLoadedFileLocation ), "%s :: %s", file->unzFilePath, filename ); snprintf( g_strLoadedFileLocation, sizeof( g_strLoadedFileLocation ), "%s :: %s", file.pak.unzFilePath.c_str(), filename );
memcpy( file->zipfile, &file->zipinfo, sizeof( unz_s ) ); unzFile zipfile = file.pak.zipfile;
*(unz_s*)zipfile = file.zipinfo;
if ( unzOpenCurrentFile( file->zipfile ) != UNZ_OK ) { if ( unzOpenCurrentFile( zipfile ) != UNZ_OK ) {
return -1; return -1;
} }
*bufferptr = safe_malloc( file->size + 1 ); *bufferptr = safe_malloc( file.size + 1 );
// we need to end the buffer with a 0 // we need to end the buffer with a 0
( (char*) ( *bufferptr ) )[file->size] = 0; ( (char*) ( *bufferptr ) )[file.size] = 0;
i = unzReadCurrentFile( file->zipfile, *bufferptr, file->size ); i = unzReadCurrentFile( zipfile, *bufferptr, file.size );
unzCloseCurrentFile( file->zipfile ); unzCloseCurrentFile( zipfile );
if ( i < 0 ) { if ( i < 0 ) {
return -1; return -1;
} }
else{ else{
return file->size; return file.size;
} }
} }
@ -408,7 +407,6 @@ int vfsLoadFile( const char *filename, void **bufferptr, int index ){
bool vfsPackFile( const char *filename, const char *packname, const int compLevel ){ bool vfsPackFile( const char *filename, const char *packname, const int compLevel ){
int i; int i;
char tmp[NAME_MAX], fixed[NAME_MAX]; char tmp[NAME_MAX], fixed[NAME_MAX];
GSList *lst;
byte *bufferptr = NULL; byte *bufferptr = NULL;
strcpy( fixed, filename ); strcpy( fixed, filename );
@ -452,32 +450,31 @@ bool vfsPackFile( const char *filename, const char *packname, const int compLeve
} }
} }
for ( lst = g_pakFiles; lst != NULL; lst = g_slist_next( lst ) ) for ( const VFS_PAKFILE& file : g_pakFiles )
{ {
VFS_PAKFILE* file = (VFS_PAKFILE*)lst->data; if ( !strEqual( file.name.c_str(), fixed ) ) {
if ( !strEqual( file->name, fixed ) ) {
continue; continue;
} }
memcpy( file->zipfile, &file->zipinfo, sizeof( unz_s ) ); unzFile zipfile = file.pak.zipfile;
*(unz_s*)zipfile = file.zipinfo;
if ( unzOpenCurrentFile( file->zipfile ) != UNZ_OK ) { if ( unzOpenCurrentFile( zipfile ) != UNZ_OK ) {
return false; return false;
} }
bufferptr = safe_malloc( file->size + 1 ); bufferptr = safe_malloc( file.size + 1 );
// we need to end the buffer with a 0 // we need to end the buffer with a 0
bufferptr[file->size] = 0; bufferptr[file.size] = 0;
i = unzReadCurrentFile( file->zipfile, bufferptr, file->size ); i = unzReadCurrentFile( zipfile, bufferptr, file.size );
unzCloseCurrentFile( file->zipfile ); unzCloseCurrentFile( zipfile );
if ( i < 0 ) { if ( i < 0 ) {
return false; return false;
} }
else{ else{
mz_bool success = MZ_TRUE; mz_bool success = MZ_TRUE;
success &= mz_zip_add_mem_to_archive_file_in_place_with_time( packname, filename, bufferptr, i, 0, 0, compLevel, file->zipinfo.cur_file_info.dosDate ); success &= mz_zip_add_mem_to_archive_file_in_place_with_time( packname, filename, bufferptr, i, 0, 0, compLevel, file.zipinfo.cur_file_info.dosDate );
if ( !success ){ if ( !success ){
Error( "Failed creating zip archive \"%s\"!\n", packname ); Error( "Failed creating zip archive \"%s\"!\n", packname );
} }