Support for comments in enums. Improve comment handling in C++ parser. Improve doxygen reference. Update interrogate interface. (Addendum to interrogatedb v2.3.)

dtool/src/cppparser/cppBison.yxx

@@ -1857,32 +1857,34 @@ named_enum:
 }
         ;
 
-enum_body:
+enum_body_trailing_comma:
         empty
-        | enum_body_no_trailing_comma
-        | enum_body_no_trailing_comma ','
-        ;
-
-enum_body_no_trailing_comma:
-        name
+        | enum_body_trailing_comma name ','
 {
   assert(current_enum != NULL);
-  current_enum->add_element($1->get_simple_name(), current_scope);
+  CPPInstance *inst = current_enum->add_element($2->get_simple_name());
+  current_scope->add_enum_value(inst, current_lexer, @2);
 }
-        | name '=' const_expr
+        | enum_body_trailing_comma name '=' const_expr ','
 {
   assert(current_enum != NULL);
-  current_enum->add_element($1->get_simple_name(), current_scope, $3);
-}
-        | enum_body_no_trailing_comma ',' name
+  CPPInstance *inst = current_enum->add_element($2->get_simple_name(), $4);
+  current_scope->add_enum_value(inst, current_lexer, @2);
+};
+
+enum_body:
+        enum_body_trailing_comma
+        | enum_body_trailing_comma name
 {
   assert(current_enum != NULL);
-  current_enum->add_element($3->get_simple_name(), current_scope);
+  CPPInstance *inst = current_enum->add_element($2->get_simple_name());
+  current_scope->add_enum_value(inst, current_lexer, @2);
 }
-        | enum_body_no_trailing_comma ',' name '=' const_expr
+        | enum_body_trailing_comma name '=' const_expr
 {
   assert(current_enum != NULL);
-  current_enum->add_element($3->get_simple_name(), current_scope, $5);
+  CPPInstance *inst = current_enum->add_element($2->get_simple_name(), $4);
+  current_scope->add_enum_value(inst, current_lexer, @2);
 }
         ;
 

dtool/src/cppparser/cppEnumType.cxx

@@ -38,8 +38,8 @@ CPPEnumType(CPPIdentifier *ident, CPPScope *current_scope,
 //       Access: Public
 //  Description:
 ////////////////////////////////////////////////////////////////////
-void CPPEnumType::
-add_element(const string &name, CPPScope *scope, CPPExpression *value) {
+CPPInstance *CPPEnumType::
+add_element(const string &name, CPPExpression *value) {
   CPPType *type =
     CPPType::new_type(new CPPSimpleType(CPPSimpleType::T_int,
                                         CPPSimpleType::F_unsigned));
@@ -47,7 +47,7 @@ add_element(const string &name, CPPScope *scope, CPPExpression *value) {
   CPPInstance *inst = new CPPInstance(type, ident);
   inst->_initializer = value;
   _elements.push_back(inst);
-  scope->add_enum_value(inst);
+  return inst;
 }
 
 ////////////////////////////////////////////////////////////////////

dtool/src/cppparser/cppEnumType.h

@@ -35,8 +35,8 @@ public:
   CPPEnumType(CPPIdentifier *ident, CPPScope *current_scope,
               const CPPFile &file);
 
-  void add_element(const string &name, CPPScope *scope,
-                   CPPExpression *value = (CPPExpression *)NULL);
+  CPPInstance *add_element(const string &name,
+                           CPPExpression *value = (CPPExpression *)NULL);
 
   virtual bool is_incomplete() const;
 

dtool/src/cppparser/cppPreprocessor.cxx

@@ -528,6 +528,34 @@ get_comment_before(int line, CPPFile file) {
   return (CPPCommentBlock *)NULL;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: CPPPreprocessor::get_comment_on
+//       Access: Public
+//  Description: Returns the CPPCommentBlock that starts on the
+//               indicated line, if any.  If there is no such
+//               comment, returns NULL.
+////////////////////////////////////////////////////////////////////
+CPPCommentBlock *CPPPreprocessor::
+get_comment_on(int line, CPPFile file) {
+  CPPComments::reverse_iterator ci;
+  ci = _comments.rbegin();
+
+  while (ci != _comments.rend()) {
+    CPPCommentBlock *comment = (*ci);
+    if (comment->_file == file) {
+      if (comment->_line_number == line) {
+        return comment;
+      } else if (comment->_line_number < line) {
+        return (CPPCommentBlock *)NULL;
+      }
+    }
+
+    ++ci;
+  }
+
+  return (CPPCommentBlock *)NULL;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: CPPPreprocessor::init_cpp
 //       Access: Protected
@@ -891,6 +919,10 @@ internal_get_next_token() {
     }
   }
 
+  // We skip whitespace here again, so that we can read any comments
+  // after this point before we parse this line.
+  _last_c = skip_whitespace(_last_c);
+
   return CPPToken(c, first_line, first_col, first_file);
 }
 
@@ -1019,11 +1051,19 @@ skip_cpp_comment(int c) {
   if (_save_comments) {
     CPPCommentBlock *comment;
 
-    if (_last_cpp_comment) {
+    int line_number = get_line_number();
+    if (c == '\n') {
+      // We have to subtract one from the line number as we just
+      // fetched a newline.
+      --line_number;
+    }
+
+    if (_last_cpp_comment && !_comments.empty() &&
+        _comments.back()->_last_line >= line_number - 1) {
       // If the last non-whitespace character read was also part of a
       // C++ comment, then this is just a continuation of that comment
-      // block.
-      assert(!_comments.empty());
+      // block.  However, if there was a line without comment in between,
+      // it starts a new block anyway.
       comment = _comments.back();
       assert(!comment->_c_style);
       comment->_comment += "//";
@@ -1033,8 +1073,8 @@ skip_cpp_comment(int c) {
       comment = new CPPCommentBlock;
 
       comment->_file = get_file();
-      comment->_line_number = get_line_number();
-      comment->_last_line = get_line_number();
+      comment->_line_number = line_number;
+      comment->_last_line = line_number;
       comment->_col_number = get_col_number() - 2;
       comment->_c_style = false;
       comment->_comment = "//";
@@ -1048,7 +1088,7 @@ skip_cpp_comment(int c) {
     }
 
     comment->_comment += '\n';
-    comment->_last_line = get_line_number();
+    comment->_last_line = line_number;
 
     _last_cpp_comment = true;
 

dtool/src/cppparser/cppPreprocessor.h

@@ -64,6 +64,7 @@ public:
              CPPFile file = CPPFile());
 
   CPPCommentBlock *get_comment_before(int line, CPPFile file);
+  CPPCommentBlock *get_comment_on(int line, CPPFile file);
 
   int get_warning_count() const;
   int get_error_count() const;
@@ -75,7 +76,7 @@ public:
   DSearchPath _quote_include_path;
   DSearchPath _angle_include_path;
   bool _noangles;
-  
+
   CPPComments _comments;
 
   typedef set<CPPFile> ParsedFiles;

dtool/src/cppparser/cppScope.cxx

@@ -145,9 +145,21 @@ add_declaration(CPPDeclaration *decl, CPPScope *global_scope,
 //  Description:
 ////////////////////////////////////////////////////////////////////
 void CPPScope::
-add_enum_value(CPPInstance *inst) {
+add_enum_value(CPPInstance *inst, CPPPreprocessor *preprocessor,
+               const cppyyltype &pos) {
   inst->_vis = _current_vis;
 
+  if (inst->_leading_comment == (CPPCommentBlock *)NULL) {
+    // Same-line comment?
+    inst->_leading_comment =
+      preprocessor->get_comment_on(pos.first_line, pos.file);
+
+    if (inst->_leading_comment == (CPPCommentBlock *)NULL) {
+      inst->_leading_comment =
+        preprocessor->get_comment_before(pos.first_line, pos.file);
+    }
+  }
+
   string name = inst->get_simple_name();
   if (!name.empty()) {
     _enum_values[name] = inst;

dtool/src/cppparser/cppScope.h

@@ -63,7 +63,9 @@ public:
   virtual void add_declaration(CPPDeclaration *decl, CPPScope *global_scope,
                                CPPPreprocessor *preprocessor,
                                const cppyyltype &pos);
-  virtual void add_enum_value(CPPInstance *inst);
+  virtual void add_enum_value(CPPInstance *inst,
+                              CPPPreprocessor *preprocessor,
+                              const cppyyltype &pos);
   virtual void define_extension_type(CPPExtensionType *type);
   virtual void define_namespace(CPPNamespace *scope);
   virtual void add_using(CPPUsing *using_decl, CPPScope *global_scope,

dtool/src/cppparser/cppTemplateScope.cxx

@@ -51,10 +51,11 @@ add_declaration(CPPDeclaration *decl, CPPScope *global_scope,
 //  Description:
 ////////////////////////////////////////////////////////////////////
 void CPPTemplateScope::
-add_enum_value(CPPInstance *inst) {
+add_enum_value(CPPInstance *inst, CPPPreprocessor *preprocessor,
+               const cppyyltype &pos) {
   inst->_template_scope = this;
   assert(_parent_scope != NULL);
-  _parent_scope->add_enum_value(inst);
+  _parent_scope->add_enum_value(inst, preprocessor, pos);
 }
 
 ////////////////////////////////////////////////////////////////////

dtool/src/cppparser/cppTemplateScope.h

@@ -36,7 +36,9 @@ public:
   virtual void add_declaration(CPPDeclaration *decl, CPPScope *global_scope,
                                CPPPreprocessor *preprocessor,
                                const cppyyltype &pos);
-  virtual void add_enum_value(CPPInstance *inst);
+  virtual void add_enum_value(CPPInstance *inst,
+                              CPPPreprocessor *preprocessor,
+                              const cppyyltype &pos);
   virtual void define_extension_type(CPPExtensionType *type);
   virtual void define_namespace(CPPNamespace *scope);
   virtual void add_using(CPPUsing *using_decl, CPPScope *global_scope,

dtool/src/interrogate/interfaceMakerPython.cxx

@@ -36,7 +36,8 @@ InterfaceMakerPython(InterrogateModuleDef *def) :
 void InterfaceMakerPython::
 write_includes(ostream &out) {
   InterfaceMaker::write_includes(out);
-  out << "#undef _POSIX_C_SOURCE\n\n"
+  out << "#undef _POSIX_C_SOURCE\n"
+      << "#define PY_SSIZE_T_CLEAN 1\n\n"
       << "#if PYTHON_FRAMEWORK\n"
       << "  #include \"Python/Python.h\"\n"
       << "#else\n"

dtool/src/interrogate/interfaceMakerPythonObj.cxx

@@ -360,21 +360,21 @@ write_function_instance(ostream &out, int indent_level,
     CPPType *orig_type = remap->_parameters[pn]._remap->get_orig_type();
     CPPType *type = remap->_parameters[pn]._remap->get_new_type();
     string param_name = remap->get_parameter_name(pn);
-    
+
     // This is the string to convert our local variable to the
     // appropriate C++ type.  Normally this is just a cast.
     string pexpr_string =
       "(" + type->get_local_name(&parser) + ")" + param_name;
-    
+
     if (remap->_parameters[pn]._remap->new_type_is_atomic_string()) {
       if (TypeManager::is_char_pointer(orig_type)) {
         out << "char *" << param_name;
         format_specifiers += "s";
         parameter_list += ", &" + param_name;
-        
+
       } else {
         out << "char *" << param_name
-            << "_str; int " << param_name << "_len";
+            << "_str; Py_ssize_t " << param_name << "_len";
         format_specifiers += "s#";
         parameter_list += ", &" + param_name
           + "_str, &" + param_name + "_len";
@@ -383,7 +383,7 @@ write_function_instance(ostream &out, int indent_level,
           param_name + "_len)";
       }
       expected_params += "string";
-      
+
     } else if (TypeManager::is_bool(type)) {
       out << "PyObject *" << param_name;
       format_specifiers += "O";

dtool/src/interrogate/interfaceMakerPythonSimple.cxx

@@ -270,21 +270,21 @@ void InterfaceMakerPythonSimple::write_function_instance(ostream &out, Interface
     CPPType *orig_type = remap->_parameters[pn]._remap->get_orig_type();
     CPPType *type = remap->_parameters[pn]._remap->get_new_type();
     string param_name = remap->get_parameter_name(pn);
-    
+
     // This is the string to convert our local variable to the
     // appropriate C++ type.  Normally this is just a cast.
     string pexpr_string =
       "(" + type->get_local_name(&parser) + ")" + param_name;
-    
+
     if (remap->_parameters[pn]._remap->new_type_is_atomic_string()) {
       if (TypeManager::is_char_pointer(orig_type)) {
         out << "char *" << param_name;
         format_specifiers += "s";
         parameter_list += ", &" + param_name;
-        
+
       } else if (TypeManager::is_wstring(orig_type)) {
         out << "Py_UNICODE *" << param_name
-            << "_str; int " << param_name << "_len";
+            << "_str; Py_ssize_t " << param_name << "_len";
         format_specifiers += "u#";
         parameter_list += ", &" + param_name
           + "_str, &" + param_name + "_len";
@@ -294,7 +294,7 @@ void InterfaceMakerPythonSimple::write_function_instance(ostream &out, Interface
 
       } else {
         out << "char *" << param_name
-            << "_str; int " << param_name << "_len";
+            << "_str; Py_ssize_t " << param_name << "_len";
         format_specifiers += "s#";
         parameter_list += ", &" + param_name
           + "_str, &" + param_name + "_len";
@@ -302,7 +302,7 @@ void InterfaceMakerPythonSimple::write_function_instance(ostream &out, Interface
           param_name + "_str, " +
           param_name + "_len)";
       }
-      
+
     } else if (TypeManager::is_bool(type)) {
       out << "PyObject *" << param_name;
       format_specifiers += "O";

dtool/src/interrogate/interrogateBuilder.cxx

@@ -1358,7 +1358,7 @@ scan_element(CPPInstance *element, CPPStructType *struct_type,
   ielement._name = element->get_local_name(scope);
   ielement._scoped_name = descope(element->get_local_name(&parser));
 
-  // See if there happens to be a comment before the MAKE_PROPERTY macro.
+  // See if there happens to be a comment before the element.
   if (element->_leading_comment != (CPPCommentBlock *)NULL) {
     ielement._comment = trim_blanks(element->_leading_comment->_comment);
   }
@@ -1838,7 +1838,7 @@ get_make_property(CPPMakeProperty *make_property, CPPStructType *struct_type) {
   iproperty._getter = get_function(getter, "", struct_type,
                                    struct_type->get_scope(), 0);
 
-  // See if there happens to be a comment before the element.
+  // See if there happens to be a comment before the MAKE_PROPERTY macro.
   if (make_property->_leading_comment != (CPPCommentBlock *)NULL) {
     iproperty._comment = trim_blanks(make_property->_leading_comment->_comment);
   }
@@ -2661,6 +2661,10 @@ define_enum_type(InterrogateType &itype, CPPEnumType *cpptype) {
     evalue._name = element->get_simple_name();
     evalue._scoped_name = descope(element->get_local_name(&parser));
 
+    if (element->_leading_comment != (CPPCommentBlock *)NULL) {
+      evalue._comment = trim_blanks(element->_leading_comment->_comment);
+    }
+
     if (element->_initializer != (CPPExpression *)NULL) {
       CPPExpression::Result result = element->_initializer->evaluate();
       next_value = result.as_integer();

dtool/src/interrogatedb/interrogateType.I

@@ -263,6 +263,19 @@ get_enum_value_scoped_name(int n) const {
   return _empty_string;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: InterrogateType::get_enum_value_comment
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+INLINE const string &InterrogateType::
+get_enum_value_comment(int n) const {
+  if (n >= 0 && n < (int)_enum_values.size()) {
+    return _enum_values[n]._comment;
+  }
+  return _empty_string;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: InterrogateType::get_enum_value
 //       Access: Public

dtool/src/interrogatedb/interrogateType.cxx

@@ -15,6 +15,7 @@
 #include "interrogateType.h"
 #include "indexRemapper.h"
 #include "interrogate_datafile.h"
+#include "interrogateDatabase.h"
 
 #include <algorithm>
 
@@ -76,6 +77,7 @@ void InterrogateType::EnumValue::
 output(ostream &out) const {
   idf_output_string(out, _name);
   idf_output_string(out, _scoped_name);
+  idf_output_string(out, _comment, '\n');
   out << _value;
 }
 
@@ -88,6 +90,9 @@ void InterrogateType::EnumValue::
 input(istream &in) {
   idf_input_string(in, _name);
   idf_input_string(in, _scoped_name);
+  if (InterrogateDatabase::get_file_minor_version() >= 3) {
+    idf_input_string(in, _comment);
+  }
   in >> _value;
 }
 

dtool/src/interrogatedb/interrogateType.h

@@ -66,6 +66,7 @@ public:
   INLINE int number_of_enum_values() const;
   INLINE const string &get_enum_value_name(int n) const;
   INLINE const string &get_enum_value_scoped_name(int n) const;
+  INLINE const string &get_enum_value_comment(int n) const;
   INLINE int get_enum_value(int n) const;
 
   INLINE bool is_struct() const;
@@ -189,6 +190,7 @@ public:
 
     string _name;
     string _scoped_name;
+    string _comment;
     int _value;
   };
 

dtool/src/interrogatedb/interrogate_interface.cxx

@@ -120,6 +120,18 @@ interrogate_element_scoped_name(ElementIndex element) {
   return InterrogateDatabase::get_ptr()->get_element(element).get_scoped_name().c_str();
 }
 
+bool
+interrogate_element_has_comment(ElementIndex element) {
+  //cerr << "interrogate_element_has_comment(" << element << ")\n";
+  return InterrogateDatabase::get_ptr()->get_element(element).has_comment();
+}
+
+const char *
+interrogate_element_comment(ElementIndex element) {
+  //cerr << "interrogate_element_comment(" << element << ")\n";
+  return InterrogateDatabase::get_ptr()->get_element(element).get_comment().c_str();
+}
+
 ElementIndex
 interrogate_get_element_by_name(const char *element_name) {
   //cerr << "interrogate_get_element_by_name(" << element_name << ")\n";
@@ -631,6 +643,12 @@ interrogate_type_enum_value_scoped_name(TypeIndex type, int n) {
   return InterrogateDatabase::get_ptr()->get_type(type).get_enum_value_scoped_name(n).c_str();
 }
 
+const char *
+interrogate_type_enum_value_comment(TypeIndex type, int n) {
+  //cerr << "interrogate_type_enum_value_comment(" << type << ", " << n << ")\n";
+  return InterrogateDatabase::get_ptr()->get_type(type).get_enum_value_comment(n).c_str();
+}
+
 int
 interrogate_type_enum_value(TypeIndex type, int n) {
   //cerr << "interrogate_type_enum_value(" << type << ", " << n << ")\n";

dtool/src/interrogatedb/interrogate_interface.h

@@ -138,6 +138,8 @@ EXPCL_DTOOLCONFIG int interrogate_manifest_get_int_value(ManifestIndex manifest)
 
 EXPCL_DTOOLCONFIG const char *interrogate_element_name(ElementIndex element);
 EXPCL_DTOOLCONFIG const char *interrogate_element_scoped_name(ElementIndex element);
+EXPCL_DTOOLCONFIG bool interrogate_element_has_comment(ElementIndex element);
+EXPCL_DTOOLCONFIG const char *interrogate_element_comment(ElementIndex element);
 EXPCL_DTOOLCONFIG ElementIndex interrogate_get_element_by_name(const char *element_name);
 EXPCL_DTOOLCONFIG ElementIndex interrogate_get_element_by_scoped_name(const char *element_name);
 
@@ -454,6 +456,7 @@ EXPCL_DTOOLCONFIG bool interrogate_type_is_enum(TypeIndex type);
 EXPCL_DTOOLCONFIG int interrogate_type_number_of_enum_values(TypeIndex type);
 EXPCL_DTOOLCONFIG const char *interrogate_type_enum_value_name(TypeIndex type, int n);
 EXPCL_DTOOLCONFIG const char *interrogate_type_enum_value_scoped_name(TypeIndex type, int n);
+EXPCL_DTOOLCONFIG const char *interrogate_type_enum_value_comment(TypeIndex type, int n);
 EXPCL_DTOOLCONFIG int interrogate_type_enum_value(TypeIndex type, int n);
 
 // If none of the above is true, the type is some extension type.  It