quick/netradiant-custom
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor tokenizing

Browse Source
This commit is contained in:
Garux 2021-10-06 23:37:12 +03:00
parent fa1933fff5
commit 71c72527af
7 changed files with 92 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/iscriplib.h
View File

@ -39,7 +39,7 @@ public:
virtual void ungetToken() = 0;
virtual std::size_t getLine() const = 0;
virtual std::size_t getColumn() const = 0;
virtual bool bufferContains( const char* str ) = 0;
virtual bool bufferContains( const char* str ) const = 0;
};
class TextInputStream;

2
libs/script/scripttokeniser.h
View File

@ -341,7 +341,7 @@ public:
std::size_t getColumn() const {
return m_scriptcolumn;
}
bool bufferContains( const char* str ){
bool bufferContains( const char* str ) const {
return m_istream.bufferContains( str );
}
};

2
libs/stream/textstream.h
View File

@ -389,7 +389,7 @@ public:
c = *m_cur++;
return true;
}
bool bufferContains( const char* str ) const{
bool bufferContains( const char* str ) const {
return ( std::search( m_cur, m_end, str, str + strlen( str ) ) != m_end ) ||
( std::search( m_buffer2, m_end2, str, str + strlen( str ) ) != m_end2 );
}

216
tools/quake3/common/scriplib.cpp
View File

@ -27,6 +27,7 @@
#include "qpathops.h"
#include "scriplib.h"
#include "vfs.h"
#include <list>
/*
=============================================================================
@ -38,19 +39,32 @@
struct script_t
{
char filename[1024];
CopiedString filename;
char *buffer;
const char *script_p, *end_p;
const char *it, *end;
int line;
/* buffer is optional: != nullptr == own it */
script_t( const char *filename, char *buffer, char *start, int size ) :
filename( filename ),
buffer( buffer ),
it( start ),
end( start + size ),
line( 1 )
{}
script_t() = delete;
script_t( const script_t& ) = delete;
script_t( script_t&& ) noexcept = delete;
script_t& operator=( const script_t& ) = delete;
script_t& operator=( script_t&& ) noexcept = delete;
~script_t(){
free( buffer );
}
};
#define MAX_INCLUDES 8
script_t scriptstack[MAX_INCLUDES];
script_t *script;
int scriptline;
std::list<script_t> scriptstack;
int scriptline;
char token[MAXTOKEN];
bool endofscript;
bool tokenready; // only true if UnGetToken was just called
/*
@ -58,35 +72,23 @@ bool tokenready; // only true if UnGetToken was just called
AddScriptToStack
==============
*/
void AddScriptToStack( const char *filename, int index ){
int size;
static void AddScriptToStack( const char *filename, int index, bool verbose ){
void* buffer;
script++;
if ( script == &scriptstack[MAX_INCLUDES] ) {
Error( "script file exceeded MAX_INCLUDES" );
}
strcpy( script->filename, ExpandPath( filename ) );
size = vfsLoadFile( script->filename, &buffer, index );
const int size = vfsLoadFile( filename, &buffer, index );
if ( size == -1 ) {
Sys_FPrintf( SYS_WRN, "Script file %s was not found\n", script->filename );
script--;
Sys_FPrintf( SYS_WRN, "Script file %s was not found\n", filename );
}
else
{
if ( index > 0 ) {
Sys_Printf( "entering %s (%d)\n", script->filename, index + 1 );
}
else{
Sys_Printf( "entering %s\n", script->filename );
if( verbose ){
if ( index > 0 )
Sys_Printf( "entering %s (%d)\n", filename, index + 1 );
else
Sys_Printf( "entering %s\n", filename );
}
script->buffer = void_ptr( buffer );
script->line = 1;
script->script_p = script->buffer;
script->end_p = script->buffer + size;
scriptstack.emplace_back( filename, void_ptr( buffer ), void_ptr( buffer ), size );
}
}
@ -96,40 +98,9 @@ void AddScriptToStack( const char *filename, int index ){
LoadScriptFile
==============
*/
void LoadScriptFile( const char *filename, int index ){
script = scriptstack;
AddScriptToStack( filename, index );
endofscript = false;
tokenready = false;
}
/* &unload current; for autopacker */
void SilentLoadScriptFile( const char *filename, int index ){
int size;
if ( script->buffer != NULL && !endofscript ) {
free( script->buffer );
script->buffer = NULL;
}
script = scriptstack;
script++;
if ( script == &scriptstack[MAX_INCLUDES] ) {
Error( "script file exceeded MAX_INCLUDES" );
}
strcpy( script->filename, ExpandPath( filename ) );
size = vfsLoadFile( script->filename, (void **)&script->buffer, index );
if ( size == -1 ) {
Sys_FPrintf( SYS_WRN, "Script file %s was not found\n", script->filename );
}
script->line = 1;
script->script_p = script->buffer;
script->end_p = script->buffer + size;
endofscript = false;
void LoadScriptFile( const char *filename, int index, bool verbose /* = true */ ){
scriptstack.clear();
AddScriptToStack( filename, index, verbose );
tokenready = false;
}
@ -139,19 +110,8 @@ void SilentLoadScriptFile( const char *filename, int index ){
==============
*/
void ParseFromMemory( char *buffer, int size ){
script = scriptstack;
script++;
if ( script == &scriptstack[MAX_INCLUDES] ) {
Error( "script file exceeded MAX_INCLUDES" );
}
strcpy( script->filename, "memory buffer" );
script->buffer = buffer;
script->line = 1;
script->script_p = script->buffer;
script->end_p = script->buffer + size;
endofscript = false;
scriptstack.clear();
scriptstack.emplace_back( "memory buffer", nullptr, buffer, size );
tokenready = false;
}
@ -171,35 +131,26 @@ void ParseFromMemory( char *buffer, int size ){
==============
*/
void UnGetToken( void ){
ENSURE( !tokenready && "Can't UnGetToken() twice in a row!" );
tokenready = true;
}
bool EndOfScript( bool crossline ){
static bool EndOfScript( bool crossline ){
if ( !crossline ) {
Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
}
if ( strEqual( script->filename, "memory buffer" ) ) {
endofscript = true;
return false;
}
scriptstack.pop_back();
if ( script->buffer == NULL ) {
Sys_Warning( "Attempt to free already freed script buffer\n" );
if ( scriptstack.empty() ) {
return false;
}
else{
free( script->buffer );
scriptline = scriptstack.back().line;
Sys_Printf( "returning to %s\n", scriptstack.back().filename.c_str() );
return GetToken( crossline );
}
script->buffer = NULL;
if ( script == scriptstack + 1 ) {
endofscript = true;
return false;
}
script--;
scriptline = script->line;
Sys_Printf( "returning to %s\n", script->filename );
return GetToken( crossline );
}
/*
@ -208,11 +159,8 @@ bool EndOfScript( bool crossline ){
==============
*/
bool GetToken( bool crossline ){
char *token_p;
/* ydnar: dummy testing */
if ( script == NULL || script->buffer == NULL ) {
if ( scriptstack.empty() ) {
return false;
}
@ -221,7 +169,9 @@ bool GetToken( bool crossline ){
return true;
}
if ( ( script->script_p >= script->end_p ) || ( script->script_p == NULL ) ) {
script_t& script = scriptstack.back();
if ( script.it >= script.end ) {
return EndOfScript( crossline );
}
@ -229,85 +179,85 @@ bool GetToken( bool crossline ){
// skip space
//
skipspace:
while ( script->script_p < script->end_p && *script->script_p <= 32 )
while ( script.it < script.end && *script.it <= 32 )
{
if ( *script->script_p++ == '\n' ) {
if ( *script.it++ == '\n' ) {
if ( !crossline ) {
Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
}
script->line++;
scriptline = script->line;
script.line++;
scriptline = script.line;
}
}
if ( script->script_p >= script->end_p ) {
if ( script.it >= script.end ) {
return EndOfScript( crossline );
}
// ; # // comments
if ( *script->script_p == ';' || *script->script_p == '#'
|| ( script->script_p[0] == '/' && script->script_p[1] == '/' ) ) {
if ( *script.it == ';' || *script.it == '#'
|| ( script.it[0] == '/' && script.it[1] == '/' ) ) {
if ( !crossline ) {
Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
}
while ( *script->script_p++ != '\n' )
if ( script->script_p >= script->end_p ) {
while ( *script.it++ != '\n' )
if ( script.it >= script.end ) {
return EndOfScript( crossline );
}
script->line++;
scriptline = script->line;
script.line++;
scriptline = script.line;
goto skipspace;
}
// /* */ comments
if ( script->script_p[0] == '/' && script->script_p[1] == '*' ) {
if ( script.it[0] == '/' && script.it[1] == '*' ) {
if ( !crossline ) {
Error( "Line %i is incomplete\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
}
script->script_p += 2;
while ( script->script_p[0] != '*' || script->script_p[1] != '/' )
script.it += 2;
while ( script.it[0] != '*' || script.it[1] != '/' )
{
if ( *script->script_p == '\n' ) {
script->line++;
scriptline = script->line;
if ( *script.it == '\n' ) {
script.line++;
scriptline = script.line;
}
script->script_p++;
if ( script->script_p >= script->end_p ) {
script.it++;
if ( script.it >= script.end ) {
return EndOfScript( crossline );
}
}
script->script_p += 2;
script.it += 2;
goto skipspace;
}
//
// copy token
//
token_p = token;
char *token_p = token;
if ( *script->script_p == '"' ) {
if ( *script.it == '"' ) {
// quoted token
script->script_p++;
while ( *script->script_p != '"' )
script.it++;
while ( *script.it != '"' )
{
*token_p++ = *script->script_p++;
if ( script->script_p == script->end_p ) {
*token_p++ = *script.it++;
if ( script.it == script.end ) {
break;
}
if ( token_p == &token[MAXTOKEN] ) {
if ( token_p == token + MAXTOKEN ) {
Error( "Token too large on line %i\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
}
}
script->script_p++;
script.it++;
}
else{ // regular token
while ( *script->script_p > 32 && *script->script_p != ';' )
while ( *script.it > 32 && *script.it != ';' )
{
*token_p++ = *script->script_p++;
if ( script->script_p == script->end_p ) {
*token_p++ = *script.it++;
if ( script.it == script.end ) {
break;
}
if ( token_p == &token[MAXTOKEN] ) {
if ( token_p == token + MAXTOKEN ) {
Error( "Token too large on line %i\nFile location be: %s\n", scriptline, g_strLoadedFileLocation );
}
}
@ -317,7 +267,7 @@ skipspace:
if ( strEqual( token, "$include" ) ) {
GetToken( false );
AddScriptToStack( token, 0 );
AddScriptToStack( token, 0, true );
return GetToken( crossline );
}
@ -333,10 +283,8 @@ skipspace:
==============
*/
bool TokenAvailable( void ) {
int oldLine;
/* save */
oldLine = scriptline;
const int oldLine = scriptline;
/* test */
if ( !GetToken( true ) ) {
@ -362,7 +310,7 @@ void MatchToken( const char *match ) {
GetToken( true );
if ( !strEqual( token, match ) ) {
Error( "MatchToken( \"%s\" ) failed at line %i in file %s", match, scriptline, script->filename );
Error( "MatchToken( \"%s\" ) failed at line %i in file %s", match, scriptline, g_strLoadedFileLocation );
}
}

12
tools/quake3/common/scriplib.h
View File

@ -19,23 +19,17 @@
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
// scriplib.h
#pragma once
#ifndef __CMDLIB__
#include "../common/cmdlib.h"
#endif
#include "cmdlib.h"
#define MAXTOKEN 1024
extern char token[MAXTOKEN];
extern char *scriptbuffer,*script_p,*scriptend_p;
extern int grabbed;
extern int scriptline;
extern bool endofscript;
void LoadScriptFile( const char *filename, int index );
void SilentLoadScriptFile( const char *filename, int index );
void LoadScriptFile( const char *filename, int index, bool verbose = true );
void ParseFromMemory( char *buffer, int size );
bool GetToken( bool crossline );

3
tools/quake3/q3data/q3data.cpp
View File

@ -441,8 +441,7 @@ void ParseScript( void ){
{
do
{ // look for a line starting with a $ command
GetToken( true );
if ( endofscript ) {
if ( !GetToken( true ) ) {
return;
}
if ( token[0] == '$' ) {

15
tools/quake3/q3map2/autopk3.cpp
View File

@ -378,9 +378,6 @@ int pk3BSPMain( Args& args ){
}
//Parse Shader Files
/* hack */
endofscript = true;
for ( CopiedString& file : pk3Shaderfiles ){
bool wantShaderFile = false;
const String64 *excludedByShader = nullptr;
@ -389,9 +386,7 @@ int pk3BSPMain( Args& args ){
/* load the shader */
const auto scriptFile = stream( g_game->shaderPath, '/', file );
SilentLoadScriptFile( scriptFile, 0 );
if( dbg )
Sys_Printf( "\n\tentering %s\n", file.c_str() );
LoadScriptFile( scriptFile, 0, dbg );
/* tokenize it */
/* check if shader file has to be excluded */
@ -442,7 +437,7 @@ int pk3BSPMain( Args& args ){
}
/* tokenize it again */
SilentLoadScriptFile( scriptFile, 0 );
LoadScriptFile( scriptFile, 0, false );
while ( 1 )
{
/* test for end of file */
@ -849,15 +844,11 @@ int repackBSPMain( Args& args ){
Sys_Printf( "\t\nParsing shaders....\n\n" );
StringOutputStream shaderText( 4096 );
StringOutputStream allShaders( 1048576 );
/* hack */
endofscript = true;
for ( const CopiedString& file : pk3Shaderfiles ){
/* load the shader */
const auto scriptFile = stream( g_game->shaderPath, '/', file );
SilentLoadScriptFile( scriptFile, 0 );
if ( dbg )
Sys_Printf( "\n\tentering %s\n", file.c_str() );
LoadScriptFile( scriptFile, 0, dbg );
/* tokenize it */
while ( 1 )