quick/netradiant-custom
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor tokenizing

Browse Source
This commit is contained in:
Garux 2021-10-06 23:37:12 +03:00
parent fa1933fff5
commit 71c72527af
7 changed files with 92 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/iscriplib.h
View File

@ -39,7 +39,7 @@ public:
virtual void ungetToken() = 0; virtual void ungetToken() = 0;
virtual std::size_t getLine() const = 0; virtual std::size_t getLine() const = 0;
virtual std::size_t getColumn() const = 0; virtual std::size_t getColumn() const = 0;
virtual bool bufferContains( const char* str ) = 0; virtual bool bufferContains( const char* str ) const = 0;
}; };
class TextInputStream; class TextInputStream;

2
libs/script/scripttokeniser.h
View File

@ -341,7 +341,7 @@ public:
std::size_t getColumn() const { std::size_t getColumn() const {
return m_scriptcolumn; return m_scriptcolumn;
} }
bool bufferContains( const char* str ){ bool bufferContains( const char* str ) const {
return m_istream.bufferContains( str ); return m_istream.bufferContains( str );
} }
}; };

2
libs/stream/textstream.h
View File

@ -389,7 +389,7 @@ public:
c = *m_cur++; c = *m_cur++;
return true; return true;
} }
bool bufferContains( const char* str ) const{ bool bufferContains( const char* str ) const {
return ( std::search( m_cur, m_end, str, str + strlen( str ) ) != m_end ) || return ( std::search( m_cur, m_end, str, str + strlen( str ) ) != m_end ) ||
( std::search( m_buffer2, m_end2, str, str + strlen( str ) ) != m_end2 ); ( std::search( m_buffer2, m_end2, str, str + strlen( str ) ) != m_end2 );
} }

216
tools/quake3/common/scriplib.cpp
View File

@ -27,6 +27,7 @@
#include "qpathops.h" #include "qpathops.h"
#include "scriplib.h" #include "scriplib.h"
#include "vfs.h" #include "vfs.h"
#include <list>
/* /*
============================================================================= =============================================================================
@ -38,19 +39,32 @@
struct script_t struct script_t
{ {
char filename[1024]; CopiedString filename;
char *buffer; char *buffer;
const char *script_p, *end_p; const char *it, *end;
int line; int line;
/* buffer is optional: != nullptr == own it */
script_t( const char *filename, char *buffer, char *start, int size ) :
filename( filename ),
buffer( buffer ),
it( start ),
end( start + size ),
line( 1 )
{}
script_t() = delete;
script_t( const script_t& ) = delete;
script_t( script_t&& ) noexcept = delete;
script_t& operator=( const script_t& ) = delete;
script_t& operator=( script_t&& ) noexcept = delete;
~script_t(){
free( buffer );
}
}; };
#define MAX_INCLUDES 8 std::list<script_t> scriptstack;
script_t scriptstack[MAX_INCLUDES];
script_t *script;
int scriptline;
int scriptline;
char token[MAXTOKEN]; char token[MAXTOKEN];
bool endofscript;
bool tokenready; // only true if UnGetToken was just called bool tokenready; // only true if UnGetToken was just called
/* /*
@ -58,35 +72,23 @@ bool tokenready; // only true if UnGetToken was just called
AddScriptToStack AddScriptToStack
============== ==============
*/ */
void AddScriptToStack( const char *filename, int index ){ static void AddScriptToStack( const char *filename, int index, bool verbose ){
int size;
void* buffer; void* buffer;
const int size = vfsLoadFile( filename, &buffer, index );
script++;
if ( script == &scriptstack[MAX_INCLUDES] ) {
Error( "script file exceeded MAX_INCLUDES" );
}
strcpy( script->filename, ExpandPath( filename ) );
size = vfsLoadFile( script->filename, &buffer, index );
if ( size == -1 ) { if ( size == -1 ) {
Sys_FPrintf( SYS_WRN, "Script file %s was not found\n", script->filename ); Sys_FPrintf( SYS_WRN, "Script file %s was not found\n", filename );
script--;
} }
else else
{ {
if ( index > 0 ) { if( verbose ){
Sys_Printf( "entering %s (%d)\n", script->filename, index + 1 ); if ( index > 0 )
} Sys_Printf( "entering %s (%d)\n", filename, index + 1 );
else{ else
Sys_Printf( "entering %s\n", script->filename ); Sys_Printf( "entering %s\n", filename );
} }
script->buffer = void_ptr( buffer ); scriptstack.emplace_back( filename, void_ptr( buffer ), void_ptr( buffer ), size );
script->line = 1;
script->script_p = script->buffer;
script->end_p = script->buffer + size;
} }
} }
@ -96,40 +98,9 @@ void AddScriptToStack( const char *filename, int index ){
LoadScriptFile LoadScriptFile
============== ==============
*/ */
void LoadScriptFile( const char *filename, int index ){ void LoadScriptFile( const char *filename, int index, bool verbose /* = true */ ){
script = scriptstack; scriptstack.clear();
AddScriptToStack( filename, index ); AddScriptToStack( filename, index, verbose );
endofscript = false;
tokenready = false;
}
/* &unload current; for autopacker */
void SilentLoadScriptFile( const char *filename, int index ){
int size;
if ( script->buffer != NULL && !endofscript ) {
free( script->buffer );
script->buffer = NULL;
}
script = scriptstack;
script++;
if ( script == &scriptstack[MAX_INCLUDES] ) {
Error( "script file exceeded MAX_INCLUDES" );
}
strcpy( script->filename, ExpandPath( filename ) );
size = vfsLoadFile( script->filename, (void **)&script->buffer, index );
if ( size == -1 ) {
Sys_FPrintf( SYS_WRN, "Script file %s was not found\n", script->filename );
}
script->line = 1;
script->script_p = script->buffer;
script->end_p = script->buffer + size;
endofscript = false;
tokenready = false; tokenready = false;
} }
@ -139,19 +110,8 @@ void SilentLoadScriptFile( const char *filename, int index ){
============== ==============
*/ */
void ParseFromMemory( char *buffer, int size ){ void ParseFromMemory( char *buffer, int size ){
script = scriptstack; scriptstack.clear();
script++; scriptstack.emplace_back( "memory buffer", nullptr, buffer, size );
if ( script == &scriptstack[MAX_INCLUDES] ) {
Error( "script file exceeded MAX_INCLUDES" );
}
strcpy( script->filename, "memory buffer" );
script->buffer = buffer;
script->line = 1;
script->script_p = script->buffer;
script->end_p = script->buffer + size;
endofscript = false;
tokenready = false; tokenready = false;
} }
@ -171,35 +131,26 @@ void ParseFromMemory( char *buffer, int size ){
============== ==============
*/ */
void UnGetToken( void ){ void UnGetToken( void ){
ENSURE( !tokenready && "Can't UnGetToken() twice in a row!" );
tokenready = true; tokenready = true;
} }
bool EndOfScript( bool crossline ){ static bool EndOfScript( bool crossline ){
if ( !crossline ) { if ( !crossline ) {
Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation ); Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
} }
if ( strEqual( script->filename, "memory buffer" ) ) { scriptstack.pop_back();
endofscript = true;
return false;
}
if ( script->buffer == NULL ) { if ( scriptstack.empty() ) {
Sys_Warning( "Attempt to free already freed script buffer\n" ); return false;
} }
else{ else{
free( script->buffer ); scriptline = scriptstack.back().line;
Sys_Printf( "returning to %s\n", scriptstack.back().filename.c_str() );
return GetToken( crossline );
} }
script->buffer = NULL;
if ( script == scriptstack + 1 ) {
endofscript = true;
return false;
}
script--;
scriptline = script->line;
Sys_Printf( "returning to %s\n", script->filename );
return GetToken( crossline );
} }
/* /*
@ -208,11 +159,8 @@ bool EndOfScript( bool crossline ){
============== ==============
*/ */
bool GetToken( bool crossline ){ bool GetToken( bool crossline ){
char *token_p;
/* ydnar: dummy testing */ /* ydnar: dummy testing */
if ( script == NULL || script->buffer == NULL ) { if ( scriptstack.empty() ) {
return false; return false;
} }
@ -221,7 +169,9 @@ bool GetToken( bool crossline ){
return true; return true;
} }
if ( ( script->script_p >= script->end_p ) || ( script->script_p == NULL ) ) { script_t& script = scriptstack.back();
if ( script.it >= script.end ) {
return EndOfScript( crossline ); return EndOfScript( crossline );
} }
@ -229,85 +179,85 @@ bool GetToken( bool crossline ){
// skip space // skip space
// //
skipspace: skipspace:
while ( script->script_p < script->end_p && *script->script_p <= 32 ) while ( script.it < script.end && *script.it <= 32 )
{ {
if ( *script->script_p++ == '\n' ) { if ( *script.it++ == '\n' ) {
if ( !crossline ) { if ( !crossline ) {
Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation ); Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
} }
script->line++; script.line++;
scriptline = script->line; scriptline = script.line;
} }
} }
if ( script->script_p >= script->end_p ) { if ( script.it >= script.end ) {
return EndOfScript( crossline ); return EndOfScript( crossline );
} }
// ; # // comments // ; # // comments
if ( *script->script_p == ';' || *script->script_p == '#' if ( *script.it == ';' || *script.it == '#'
|| ( script->script_p[0] == '/' && script->script_p[1] == '/' ) ) { || ( script.it[0] == '/' && script.it[1] == '/' ) ) {
if ( !crossline ) { if ( !crossline ) {
Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation ); Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
} }
while ( *script->script_p++ != '\n' ) while ( *script.it++ != '\n' )
if ( script->script_p >= script->end_p ) { if ( script.it >= script.end ) {
return EndOfScript( crossline ); return EndOfScript( crossline );
} }
script->line++; script.line++;
scriptline = script->line; scriptline = script.line;
goto skipspace; goto skipspace;
} }
// /* */ comments // /* */ comments
if ( script->script_p[0] == '/' && script->script_p[1] == '*' ) { if ( script.it[0] == '/' && script.it[1] == '*' ) {
if ( !crossline ) { if ( !crossline ) {
Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation ); Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
} }
script->script_p += 2; script.it += 2;
while ( script->script_p[0] != '*' || script->script_p[1] != '/' ) while ( script.it[0] != '*' || script.it[1] != '/' )
{ {
if ( *script->script_p == '\n' ) { if ( *script.it == '\n' ) {
script->line++; script.line++;
scriptline = script->line; scriptline = script.line;
} }
script->script_p++; script.it++;
if ( script->script_p >= script->end_p ) { if ( script.it >= script.end ) {
return EndOfScript( crossline ); return EndOfScript( crossline );
} }
} }
script->script_p += 2; script.it += 2;
goto skipspace; goto skipspace;
} }
// //
// copy token // copy token
// //
token_p = token; char *token_p = token;
if ( *script->script_p == '"' ) { if ( *script.it == '"' ) {
// quoted token // quoted token
script->script_p++; script.it++;
while ( *script->script_p != '"' ) while ( *script.it != '"' )
{ {
*token_p++ = *script->script_p++; *token_p++ = *script.it++;
if ( script->script_p == script->end_p ) { if ( script.it == script.end ) {
break; break;
} }
if ( token_p == &token[MAXTOKEN] ) { if ( token_p == token + MAXTOKEN ) {
Error( "Token too large on line %i\nFile location be: %s\n", scriptline, g_strLoadedFileLocation ); Error( "Token too large on line %i\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
} }
} }
script->script_p++; script.it++;
} }
else{ // regular token else{ // regular token
while ( *script->script_p > 32 && *script->script_p != ';' ) while ( *script.it > 32 && *script.it != ';' )
{ {
*token_p++ = *script->script_p++; *token_p++ = *script.it++;
if ( script->script_p == script->end_p ) { if ( script.it == script.end ) {
break; break;
} }
if ( token_p == &token[MAXTOKEN] ) { if ( token_p == token + MAXTOKEN ) {
Error( "Token too large on line %i\nFile location be: %s\n", scriptline, g_strLoadedFileLocation ); Error( "Token too large on line %i\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
} }
} }
@ -317,7 +267,7 @@ skipspace:
if ( strEqual( token, "$include" ) ) { if ( strEqual( token, "$include" ) ) {
GetToken( false ); GetToken( false );
AddScriptToStack( token, 0 ); AddScriptToStack( token, 0, true );
return GetToken( crossline ); return GetToken( crossline );
} }
@ -333,10 +283,8 @@ skipspace:
============== ==============
*/ */
bool TokenAvailable( void ) { bool TokenAvailable( void ) {
int oldLine;
/* save */ /* save */
oldLine = scriptline; const int oldLine = scriptline;
/* test */ /* test */
if ( !GetToken( true ) ) { if ( !GetToken( true ) ) {
@ -362,7 +310,7 @@ void MatchToken( const char *match ) {
GetToken( true ); GetToken( true );
if ( !strEqual( token, match ) ) { if ( !strEqual( token, match ) ) {
Error( "MatchToken( \"%s\" ) failed at line %i in file %s", match, scriptline, script->filename ); Error( "MatchToken( \"%s\" ) failed at line %i in file %s", match, scriptline, g_strLoadedFileLocation );
} }
} }

12
tools/quake3/common/scriplib.h
View File

@ -19,23 +19,17 @@
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/ */
// scriplib.h #pragma once
#ifndef __CMDLIB__ #include "cmdlib.h"
#include "../common/cmdlib.h"
#endif
#define MAXTOKEN 1024 #define MAXTOKEN 1024
extern char token[MAXTOKEN]; extern char token[MAXTOKEN];
extern char *scriptbuffer,*script_p,*scriptend_p;
extern int grabbed;
extern int scriptline; extern int scriptline;
extern bool endofscript;
void LoadScriptFile( const char *filename, int index ); void LoadScriptFile( const char *filename, int index, bool verbose = true );
void SilentLoadScriptFile( const char *filename, int index );
void ParseFromMemory( char *buffer, int size ); void ParseFromMemory( char *buffer, int size );
bool GetToken( bool crossline ); bool GetToken( bool crossline );

3
tools/quake3/q3data/q3data.cpp
View File

@ -441,8 +441,7 @@ void ParseScript( void ){
{ {
do do
{ // look for a line starting with a $ command { // look for a line starting with a $ command
GetToken( true ); if ( !GetToken( true ) ) {
if ( endofscript ) {
return; return;
} }
if ( token[0] == '$' ) { if ( token[0] == '$' ) {

15
tools/quake3/q3map2/autopk3.cpp
View File

@ -378,9 +378,6 @@ int pk3BSPMain( Args& args ){
} }
//Parse Shader Files //Parse Shader Files
/* hack */
endofscript = true;
for ( CopiedString& file : pk3Shaderfiles ){ for ( CopiedString& file : pk3Shaderfiles ){
bool wantShaderFile = false; bool wantShaderFile = false;
const String64 *excludedByShader = nullptr; const String64 *excludedByShader = nullptr;
@ -389,9 +386,7 @@ int pk3BSPMain( Args& args ){
/* load the shader */ /* load the shader */
const auto scriptFile = stream( g_game->shaderPath, '/', file ); const auto scriptFile = stream( g_game->shaderPath, '/', file );
SilentLoadScriptFile( scriptFile, 0 ); LoadScriptFile( scriptFile, 0, dbg );
if( dbg )
Sys_Printf( "\n\tentering %s\n", file.c_str() );
/* tokenize it */ /* tokenize it */
/* check if shader file has to be excluded */ /* check if shader file has to be excluded */
@ -442,7 +437,7 @@ int pk3BSPMain( Args& args ){
} }
/* tokenize it again */ /* tokenize it again */
SilentLoadScriptFile( scriptFile, 0 ); LoadScriptFile( scriptFile, 0, false );
while ( 1 ) while ( 1 )
{ {
/* test for end of file */ /* test for end of file */
@ -849,15 +844,11 @@ int repackBSPMain( Args& args ){
Sys_Printf( "\t\nParsing shaders....\n\n" ); Sys_Printf( "\t\nParsing shaders....\n\n" );
StringOutputStream shaderText( 4096 ); StringOutputStream shaderText( 4096 );
StringOutputStream allShaders( 1048576 ); StringOutputStream allShaders( 1048576 );
/* hack */
endofscript = true;
for ( const CopiedString& file : pk3Shaderfiles ){ for ( const CopiedString& file : pk3Shaderfiles ){
/* load the shader */ /* load the shader */
const auto scriptFile = stream( g_game->shaderPath, '/', file ); const auto scriptFile = stream( g_game->shaderPath, '/', file );
SilentLoadScriptFile( scriptFile, 0 ); LoadScriptFile( scriptFile, 0, dbg );
if ( dbg )
Sys_Printf( "\n\tentering %s\n", file.c_str() );
/* tokenize it */ /* tokenize it */
while ( 1 ) while ( 1 )