2025-07-18 05:19 CEST

View Issue Details Jump to Notes ]
IDProjectCategoryView StatusLast Update
0000538Spring engineGeneralpublic2007-05-30 15:46
ReporterM2 
Assigned Totvo 
PrioritynormalSeverityminorReproducibilityalways
StatusresolvedResolutionfixed 
Product Version 
Target VersionFixed in Version 
Summary0000538: [patch] TdfParser
Description[patch] TdfParser

In Boost 1.34 the parser no longer consumes trailing whitespace, adding ?óÔé¼?Ø>> end_p?óÔé¼?Ø to the parser grammar gives the old behaviour across all Boost versions.
Reference: http://boost.cvs.sourceforge.net/boost/boost/libs/spirit/test/bug_fixes.cpp?revision=1.12.6.1&view=markup

?óÔé¼?ô++error_it?óÔé¼?Ø reads a newline (two bytes on windows one byte on other systems) in one operation, but the counter doesn?óÔé¼Ôäót reflect this resulting in slightly wrong positions reported in the error messages or a read beyond the end of the buffer.

Removed unnecessary duplicated code.

TagsNo tags attached.
Checked infolog.txt for Errors
Attached Files
  • patch file icon TdfParser.patch (7,603 bytes) 2007-05-28 16:45 -
    Index: rts/System/FileSystem/ArchiveScanner.cpp
    ===================================================================
    --- rts/System/FileSystem/ArchiveScanner.cpp	(revision 3788)
    +++ rts/System/FileSystem/ArchiveScanner.cpp	(working copy)
    @@ -174,16 +174,16 @@
     							if (fh) {
     								int fsize = ar->FileSize(fh);
     
    -								void* buf = malloc(fsize);
    +								char* buf = SAFE_NEW char[fsize];
     								ar->ReadFile(fh, buf, fsize);
     								ar->CloseFile(fh);
     								try {
    -									TdfParser p( reinterpret_cast<char*>(buf), fsize );
    +									TdfParser p( buf, fsize );
     									ai.modData = GetModData(&p, "mod");
     								} catch (const TdfParser::parse_error&) {
     									// Silently ignore mods with parse errors
     								}
    -								free(buf);
    +								delete [] buf;
     							}
     
     						}
    Index: rts/System/tdf_grammar.hpp
    ===================================================================
    --- rts/System/tdf_grammar.hpp	(revision 3788)
    +++ rts/System/tdf_grammar.hpp	(working copy)
    @@ -99,6 +99,7 @@
                 section(self.section) 
                 | gather_junk_line  // if this rule gets hit then section did not consume everything,
              ) 
    +        >> end_p
             ;
     
           gather_junk_line = 
    Index: rts/System/TdfParser.cpp
    ===================================================================
    --- rts/System/TdfParser.cpp	(revision 3788)
    +++ rts/System/TdfParser.cpp	(working copy)
    @@ -95,40 +95,28 @@
     void TdfParser::print( std::ostream & out ) const {
       root_section.print( out );
     }
    -TdfParser::TdfParser(std::string const& filename) 
    -{
    -  LoadFile( filename );
    -}
    -void TdfParser::LoadFile(std::string const& filename)
    -{
    -  this->filename = filename;
    -	CFileHandler file(filename);
    -	if(!file.FileExists())
    -    throw content_error( ("file " + filename + " not found").c_str() );
     
    -	int size = file.FileSize();
    -  boost::scoped_array<char> filebuf( SAFE_NEW char[size+2] );
     
    -	file.Read( filebuf.get(), file.FileSize());
    -  filebuf[size] = '\0'; //append newline at end to avoid parsing error at eof
    +void TdfParser::parse_buffer( char const* buf, std::size_t size){
     
       std::list<std::string> junk_data;
       tdf_grammar grammar( &root_section, &junk_data );
       boost::spirit::parse_info<char const*> info;
       std::string message; 
    -  boost::spirit::position_iterator2<char const*> error_it( filebuf.get(), filebuf.get() + size, filename );
    +  typedef boost::spirit::position_iterator2<char const*> iterator_t;
    +  iterator_t error_it( buf, buf + size );
     
       try {
        info = boost::spirit::parse( 
    -      filebuf.get()
    -      , filebuf.get() + size
    +      buf
    +      , buf + size
           , grammar
           , space_p 
           |  comment_p("/*", "*/")           // rule for C-comments
           |  comment_p("//")
           ); 
       }
    -  catch( boost::spirit::parser_error<tdf_grammar::Errors, char *> & e ) { // thrown by assertion parsers in tdf_grammar
    +  catch( boost::spirit::parser_error<tdf_grammar::Errors, char const*> & e ) { // thrown by assertion parsers in tdf_grammar
     
         switch(e.descriptor) {
           case tdf_grammar::semicolon_expected: message = "semicolon expected"; break;
    @@ -137,7 +125,13 @@
           case tdf_grammar::brace_expected: message = "brace or further name value pairs expected"; break;
         };
     
    -    for( int i = 0, end = e.where -filebuf.get(); i != end; ++i,++error_it );
    +    std::ptrdiff_t target_pos = e.where - buf;
    +    for( int i = 1; i < target_pos; ++i) {
    +      ++error_it;
    +      if(error_it != (iterator_t(buf + i, buf + size))){
    +        ++i;
    +      }
    +    }
       }
     
       for( std::list<std::string>::const_iterator it = junk_data.begin(), e = junk_data.end(); 
    @@ -153,7 +147,13 @@
     
       // a different error might have happened:
       if(!info.full)  {
    -    for( int i = 0, end = info.stop -filebuf.get(); i != end; ++i,++error_it );
    +    std::ptrdiff_t target_pos = info.stop - buf;
    +    for( int i = 1; i < target_pos; ++i) {
    +      ++error_it;
    +      if(error_it != (iterator_t(buf + i, buf + size))){
    +        ++i;
    +      }
    +    }
     
         throw parse_error( error_it.get_currentline(), error_it.get_position().line, error_it.get_position().column, filename );
       }
    @@ -163,58 +163,32 @@
     TdfParser::TdfParser( char const* buf, std::size_t size) {
       LoadBuffer( buf, size );
     }
    -void TdfParser::LoadBuffer( char const* buf, std::size_t size)
    -{
    -  this->filename = "buffer";
    -  std::list<std::string> junk_data;
    -  tdf_grammar grammar( &root_section, &junk_data );
     
    -  std::string message; 
    +void TdfParser::LoadBuffer( char const* buf, std::size_t size){
     
    -  boost::spirit::position_iterator2<char const*> error_it( buf, buf + size, filename );
    +  this->filename = "buffer";
    +  parse_buffer( buf, size );
    +}
     
    -  boost::spirit::parse_info<char const*> info;
    -  try {
    -    info = parse( 
    -      buf
    -      , buf + size
    -      , grammar
    -      , space_p 
    -      |  comment_p("/*", "*/")           // rule for C-comments
    -      |  comment_p("//")
    -      ) ; 
    -  } 
    -  catch( boost::spirit::parser_error<tdf_grammar::Errors, char *> & e ) { // thrown by assertion parsers in tdf_grammar
     
    -    switch(e.descriptor) {
    -      case tdf_grammar::semicolon_expected: message = "semicolon expected"; break;
    -      case tdf_grammar::equals_sign_expected: message = "equals sign in name value pair expected"; break;
    -      case tdf_grammar::square_bracket_expected: message = "square bracket to close section name expected"; break;
    -      case tdf_grammar::brace_expected: message = "brace or further name value pairs expected"; break;
    -    };
    +void TdfParser::LoadFile(std::string const& filename){
     
    +  this->filename = filename;
    +  CFileHandler file(filename);
    +  if(!file.FileExists())
    +    throw content_error( ("file " + filename + " not found").c_str() );
     
    -    for( size_t  i = 0;i != size;  ++i,++error_it );
    +  int size = file.FileSize();
    +  boost::scoped_array<char> filebuf( SAFE_NEW char[size] );
     
    -  }
    +  file.Read( filebuf.get(), file.FileSize());
     
    +  parse_buffer(filebuf.get(), size);
    +}
     
    -  for( std::list<std::string>::const_iterator it = junk_data.begin(), e = junk_data.end(); 
    -      it !=e ; ++it ){
    -    std::string temp = boost::trim_copy( *it );
    -    if( ! temp.empty( ) ) {
    -      ::logOutput.Print( "Junk in "+ filename +  " :" + temp );
    -    }
    -  }
    +TdfParser::TdfParser(std::string const& filename) {
     
    -  if(!message.empty())
    -    throw parse_error( message, error_it.get_currentline(), error_it.get_position().line, error_it.get_position().column, filename );
    -  if(!info.full)
    -  {
    -    boost::spirit::position_iterator2<char const*> error_it( buf, buf + size, filename );
    -    for( size_t i = 0; i != size; ++i,++error_it );
    -    throw parse_error( error_it.get_currentline(), error_it.get_position().line, error_it.get_position().column, filename );
    -  }
    +  LoadFile(filename);
     }
     
     //find value, display messagebox if no such value found
    @@ -408,8 +382,8 @@
     {
       std::string lowerd = StringToLower(location);
     	std::vector<std::string> loclist;
    -	int start = 0;
    -	int next = 0;
    +	std::string::size_type start = 0;
    +	std::string::size_type next = 0;
     
     	while((next = lowerd.find_first_of("\\", start)) != std::string::npos)
     	{
    Index: rts/System/TdfParser.h
    ===================================================================
    --- rts/System/TdfParser.h	(revision 3788)
    +++ rts/System/TdfParser.h	(working copy)
    @@ -173,6 +173,10 @@
       std::string filename;
     
     	std::vector<std::string> GetLocationVector(std::string const& location);
    +
    +
    +	void parse_buffer( char const* buf, std::size_t size);
    +
     public:
     	float3 GetFloat3(float3 def, std::string const& location);
     };
    
    patch file icon TdfParser.patch (7,603 bytes) 2007-05-28 16:45 +

-Relationships
+Relationships

-Notes

~0000918

tvo (reporter)

committed, thanks for your contribution! (r3798)
+Notes

-Issue History
Date Modified Username Field Change
2007-05-28 16:45 M2 New Issue
2007-05-28 16:45 M2 File Added: TdfParser.patch
2007-05-30 15:46 tvo Status new => assigned
2007-05-30 15:46 tvo Assigned To => tvo
2007-05-30 15:46 tvo Status assigned => resolved
2007-05-30 15:46 tvo Resolution open => fixed
2007-05-30 15:46 tvo Note Added: 0000918
+Issue History