shader: supports preprocessed GLSL shaders in Shader.make()

Fixes #436

panda/src/gobj/shader.cxx

@@ -2468,6 +2468,96 @@ read(const ShaderFile &sfile, BamCacheRecord *record) {
   return true;
 }
 
+/**
+ * Loads the shader from the given string(s). Returns a boolean indicating
+ * success or failure.
+ */
+bool Shader::
+load(const ShaderFile &sbody, BamCacheRecord *record) {
+  _filename = ShaderFile("created-shader");
+  _fullpath = Filename();
+  _text._separate = sbody._separate;
+
+  if (sbody._separate) {
+    if (_language == SL_none) {
+      shader_cat.error()
+        << "No shader language was specified!\n";
+      return false;
+    }
+
+    if (!sbody._vertex.empty() &&
+        !do_load_source(_text._vertex, sbody._vertex, record)) {
+      return false;
+    }
+    if (!sbody._fragment.empty() &&
+        !do_load_source(_text._fragment, sbody._fragment, record)) {
+      return false;
+    }
+    if (!sbody._geometry.empty() &&
+        !do_load_source(_text._geometry, sbody._geometry, record)) {
+      return false;
+    }
+    if (!sbody._tess_control.empty() &&
+        !do_load_source(_text._tess_control, sbody._tess_control, record)) {
+      return false;
+    }
+    if (!sbody._tess_evaluation.empty() &&
+        !do_load_source(_text._tess_evaluation, sbody._tess_evaluation, record)) {
+      return false;
+    }
+    if (!sbody._compute.empty() &&
+        !do_load_source(_text._compute, sbody._compute, record)) {
+      return false;
+    }
+
+  } else {
+    if (!do_load_source(_text._shared, sbody._shared, record)) {
+      return false;
+    }
+
+    // Determine which language the shader is written in.
+    if (_language == SL_none) {
+      string header;
+      parse_init();
+      parse_line(header, true, true);
+      if (header == "//Cg") {
+        _language = SL_Cg;
+      } else {
+        shader_cat.error()
+          << "Unable to determine shader language of " << sbody._shared << "\n";
+        return false;
+      }
+    } else if (_language == SL_GLSL) {
+      shader_cat.error()
+        << "GLSL shaders must have separate shader bodies!\n";
+      return false;
+    }
+
+    // Determine which language the shader is written in.
+    if (_language == SL_Cg) {
+#ifdef HAVE_CG
+      cg_get_profile_from_header(_default_caps);
+
+      if (!cg_analyze_shader(_default_caps)) {
+        shader_cat.error()
+          << "Shader encountered an error.\n";
+        return false;
+      }
+#else
+      shader_cat.error()
+        << "Tried to load Cg shader, but no Cg support is enabled.\n";
+#endif
+    } else {
+      shader_cat.error()
+        << "Shader is not in a supported shader-language.\n";
+      return false;
+    }
+  }
+
+  _loaded = true;
+  return true;
+}
+
 /**
  * Reads the shader file from the given path into the given string.
  *
@@ -2476,37 +2566,85 @@ read(const ShaderFile &sfile, BamCacheRecord *record) {
  */
 bool Shader::
 do_read_source(string &into, const Filename &fn, BamCacheRecord *record) {
+  VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
+  PT(VirtualFile) vf = vfs->find_file(fn, get_model_path());
+  if (vf == nullptr) {
+    shader_cat.error()
+      << "Could not find shader file: " << fn << "\n";
+    return false;
+  }
+
   if (_language == SL_GLSL && glsl_preprocess) {
+    istream *source = vf->open_read_file(true);
+    if (source == nullptr) {
+      shader_cat.error()
+        << "Could not open shader file: " << fn << "\n";
+      return false;
+    }
+
     // Preprocess the GLSL file as we read it.
+    shader_cat.info()
+      << "Preprocessing shader file: " << fn << "\n";
+
     std::set<Filename> open_files;
     ostringstream sstr;
-    if (!r_preprocess_source(sstr, fn, Filename(), open_files, record)) {
+    if (!r_preprocess_source(sstr, *source, fn, vf->get_filename(), open_files, record)) {
+      vf->close_read_file(source);
       return false;
     }
+    vf->close_read_file(source);
     into = sstr.str();
 
   } else {
     shader_cat.info() << "Reading shader file: " << fn << "\n";
 
-    VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
-    PT(VirtualFile) vf = vfs->find_file(fn, get_model_path());
-    if (vf == nullptr) {
-      shader_cat.error()
-        << "Could not find shader file: " << fn << "\n";
-      return false;
-    }
-
     if (!vf->read_file(into, true)) {
       shader_cat.error()
         << "Could not read shader file: " << fn << "\n";
       return false;
     }
+  }
+
+  if (record != nullptr) {
+    record->add_dependent_file(vf);
+  }
+
+  _last_modified = std::max(_last_modified, vf->get_timestamp());
+  _source_files.push_back(vf->get_filename());
+
+  // Strip trailing whitespace.
+  while (!into.empty() && isspace(into[into.size() - 1])) {
+    into.resize(into.size() - 1);
+  }
+
+  // Except add back a newline at the end, which is needed by Intel drivers.
+  into += "\n";
+
+  return true;
+}
 
-    if (record != nullptr) {
-      record->add_dependent_file(vf);
+/**
+ * Loads the shader file from the given string into the given string,
+ * performing any pre-processing on it that may be necessary.
+ *
+ * Returns false if there was an error with this shader bad enough to consider
+ * it 'invalid'.
+ */
+bool Shader::
+do_load_source(string &into, const std::string &source, BamCacheRecord *record) {
+  if (_language == SL_GLSL && glsl_preprocess) {
+    // Preprocess the GLSL file as we read it.
+    std::set<Filename> open_files;
+    std::ostringstream sstr;
+    std::istringstream in(source);
+    if (!r_preprocess_source(sstr, in, Filename("created-shader"), Filename(),
+                             open_files, record)) {
+      return false;
     }
-    _last_modified = std::max(_last_modified, vf->get_timestamp());
-    _source_files.push_back(vf->get_filename());
+    into = sstr.str();
+
+  } else {
+    into = source;
   }
 
   // Strip trailing whitespace.
@@ -2528,10 +2666,10 @@ do_read_source(string &into, const Filename &fn, BamCacheRecord *record) {
  * recursive includes.
  */
 bool Shader::
-r_preprocess_source(ostream &out, const Filename &fn,
-                    const Filename &source_dir,
-                    std::set<Filename> &once_files,
-                    BamCacheRecord *record, int depth) {
+r_preprocess_include(ostream &out, const Filename &fn,
+                     const Filename &source_dir,
+                     std::set<Filename> &once_files,
+                     BamCacheRecord *record, int depth) {
 
   if (depth > glsl_include_recursion_limit) {
     shader_cat.error()
@@ -2549,7 +2687,7 @@ r_preprocess_source(ostream &out, const Filename &fn,
   PT(VirtualFile) vf = vfs->find_file(fn, path);
   if (vf == nullptr) {
     shader_cat.error()
-      << "Could not find shader file: " << fn << "\n";
+      << "Could not find shader include: " << fn << "\n";
     return false;
   }
 
@@ -2562,7 +2700,7 @@ r_preprocess_source(ostream &out, const Filename &fn,
   istream *source = vf->open_read_file(true);
   if (source == nullptr) {
     shader_cat.error()
-      << "Could not open shader file: " << fn << "\n";
+      << "Could not open shader include: " << fn << "\n";
     return false;
   }
 
@@ -2578,30 +2716,43 @@ r_preprocess_source(ostream &out, const Filename &fn,
   // than that, unfortunately.  Don't do this for the top-level file, though.
   // We don't want anything to get in before a potential #version directive.
   int fileno = 0;
-  if (depth > 0) {
-    fileno = 2048 + _included_files.size();
-    // Write it into the vector so that we can substitute it later when we are
-    // parsing the GLSL error log.  Don't store the full filename because it
-    // would just be too long to display.
-    _included_files.push_back(fn);
-
-    out << "#line 1 " << fileno << " // " << fn << "\n";
-    if (shader_cat.is_debug()) {
-      shader_cat.debug()
-        << "Preprocessing shader include " << fileno << ": " << fn << "\n";
-    }
-  } else {
-    shader_cat.info()
-      << "Preprocessing shader file: " << fn << "\n";
+  fileno = 2048 + _included_files.size();
+
+  // Write it into the vector so that we can substitute it later when we are
+  // parsing the GLSL error log.  Don't store the full filename because it
+  // would just be too long to display.
+  _included_files.push_back(fn);
+
+  out << "#line 1 " << fileno << " // " << fn << "\n";
+  if (shader_cat.is_debug()) {
+    shader_cat.debug()
+      << "Preprocessing shader include " << fileno << ": " << fn << "\n";
   }
 
+  bool result = r_preprocess_source(out, *source, fn, full_fn, once_files, record, fileno, depth);
+  vf->close_read_file(source);
+  return result;
+}
+
+/**
+ * Loads a given GLSL stream line by line, processing any #pragma include and
+ * once statements, as well as removing any comments.
+ *
+ * The set keeps track of which files we have already included, for checking
+ * recursive includes.
+ */
+bool Shader::
+r_preprocess_source(ostream &out, istream &in, const Filename &fn,
+                    const Filename &full_fn, std::set<Filename> &once_files,
+                    BamCacheRecord *record, int fileno, int depth) {
+
   // Iterate over the lines for things we may need to preprocess.
   string line;
   int ext_google_include = 0; // 1 = warn, 2 = enable
   int ext_google_line = 0;
   bool had_include = false;
   int lineno = 0;
-  while (std::getline(*source, line)) {
+  while (std::getline(in, line)) {
     ++lineno;
 
     if (line.empty()) {
@@ -2615,7 +2766,7 @@ r_preprocess_source(ostream &out, const Filename &fn,
       line.resize(line.size() - 1);
       string line2;
 
-      if (std::getline(*source, line2)) {
+      if (std::getline(in, line2)) {
         line += line2;
         out.put('\n');
         ++lineno;
@@ -2644,7 +2795,7 @@ r_preprocess_source(ostream &out, const Filename &fn,
       size_t block_end = line2.find("*/");
       while (block_end == string::npos) {
         // Didn't find it - look in the next line.
-        if (std::getline(*source, line2)) {
+        if (std::getline(in, line2)) {
           out.put('\n');
           ++lineno;
           block_end = line2.find("*/");
@@ -2703,7 +2854,7 @@ r_preprocess_source(ostream &out, const Filename &fn,
         }
 
         // OK, great.  Process the include.
-        if (!r_preprocess_source(out, incfn, source_dir, once_files, record, depth + 1)) {
+        if (!r_preprocess_include(out, incfn, source_dir, once_files, record, depth + 1)) {
           // An error occurred.  Pass on the failure.
           shader_cat.error(false) << "included at line "
             << lineno << " of file " << fn << ":\n  " << line << "\n";
@@ -2724,8 +2875,19 @@ r_preprocess_source(ostream &out, const Filename &fn,
           return false;
         }
 
-        once_files.insert(full_fn);
+        if (fileno == 0) {
+          shader_cat.warning()
+            << "#pragma once in main file at line "
+            << lineno << " of file " << fn
+#ifndef NDEBUG
+            << ":\n  " << line
+#endif
+            << "\n";
+        }
 
+        if (!full_fn.empty()) {
+          once_files.insert(full_fn);
+        }
       } else {
         // Forward it, the driver will ignore it if it doesn't know it.
         out << line << "\n";
@@ -2821,7 +2983,7 @@ r_preprocess_source(ostream &out, const Filename &fn,
 
       // OK, great.  Process the include.
       Filename source_dir = full_fn.get_dirname();
-      if (!r_preprocess_source(out, incfn, source_dir, once_files, record, depth + 1)) {
+      if (!r_preprocess_include(out, incfn, source_dir, once_files, record, depth + 1)) {
         // An error occurred.  Pass on the failure.
         shader_cat.error(false) << "included at line "
           << lineno << " of file " << fn << ":\n  " << line << "\n";
@@ -2865,7 +3027,6 @@ r_preprocess_source(ostream &out, const Filename &fn,
     }
   }
 
-  vf->close_read_file(source);
   return true;
 }
 
@@ -3227,28 +3388,26 @@ make(string body, ShaderLanguage lang) {
   if (cache_generated_shaders) {
     ShaderTable::const_iterator i = _make_table.find(sbody);
     if (i != _make_table.end() && (lang == SL_none || lang == i->second->_language)) {
-      return i->second;
+      // But check that someone hasn't modified its includes in the meantime.
+      if (!i->second->check_modified()) {
+        return i->second;
+      }
     }
   }
 
   PT(Shader) shader = new Shader(lang);
-  shader->_filename = ShaderFile("created-shader");
-  shader->_text = move(sbody);
-
-#ifdef HAVE_CG
-  if (lang == SL_Cg) {
-    shader->cg_get_profile_from_header(_default_caps);
-
-    if (!shader->cg_analyze_shader(_default_caps)) {
-      shader_cat.error()
-        << "Shader encountered an error.\n";
-      return nullptr;
-    }
+  if (!shader->load(sbody)) {
+    return nullptr;
   }
-#endif
 
   if (cache_generated_shaders) {
-    _make_table[shader->_text] = shader;
+    ShaderTable::const_iterator i = _make_table.find(shader->_text);
+    if (i != _make_table.end() && (lang == SL_none || lang == i->second->_language)) {
+      shader = i->second;
+    } else {
+      _make_table[shader->_text] = shader;
+    }
+    _make_table[std::move(sbody)] = shader;
   }
 
   if (dump_generated_shaders) {
@@ -3290,26 +3449,27 @@ make(ShaderLanguage lang, string vertex, string fragment, string geometry,
   if (cache_generated_shaders) {
     ShaderTable::const_iterator i = _make_table.find(sbody);
     if (i != _make_table.end() && (lang == SL_none || lang == i->second->_language)) {
-      return i->second;
+      // But check that someone hasn't modified its includes in the meantime.
+      if (!i->second->check_modified()) {
+        return i->second;
+      }
     }
   }
 
   PT(Shader) shader = new Shader(lang);
   shader->_filename = ShaderFile("created-shader");
-  shader->_text = move(sbody);
-
-#ifdef HAVE_CG
-  if (lang == SL_Cg) {
-    if (!shader->cg_analyze_shader(_default_caps)) {
-      shader_cat.error()
-        << "Shader encountered an error.\n";
-      return nullptr;
-    }
+  if (!shader->load(sbody)) {
+    return nullptr;
   }
-#endif
 
   if (cache_generated_shaders) {
-    _make_table[shader->_text] = shader;
+    ShaderTable::const_iterator i = _make_table.find(shader->_text);
+    if (i != _make_table.end() && (lang == SL_none || lang == i->second->_language)) {
+      shader = i->second;
+    } else {
+      _make_table[shader->_text] = shader;
+    }
+    _make_table[std::move(sbody)] = shader;
   }
 
   return shader;
@@ -3333,16 +3493,27 @@ make_compute(ShaderLanguage lang, string body) {
   if (cache_generated_shaders) {
     ShaderTable::const_iterator i = _make_table.find(sbody);
     if (i != _make_table.end() && (lang == SL_none || lang == i->second->_language)) {
-      return i->second;
+      // But check that someone hasn't modified its includes in the meantime.
+      if (!i->second->check_modified()) {
+        return i->second;
+      }
     }
   }
 
   PT(Shader) shader = new Shader(lang);
   shader->_filename = ShaderFile("created-shader");
-  shader->_text = move(sbody);
+  if (!shader->load(sbody)) {
+    return nullptr;
+  }
 
   if (cache_generated_shaders) {
-    _make_table[shader->_text] = shader;
+    ShaderTable::const_iterator i = _make_table.find(shader->_text);
+    if (i != _make_table.end() && (lang == SL_none || lang == i->second->_language)) {
+      shader = i->second;
+    } else {
+      _make_table[shader->_text] = shader;
+    }
+    _make_table[std::move(sbody)] = shader;
   }
 
   return shader;

panda/src/gobj/shader.h

@@ -616,11 +616,18 @@ private:
   Shader(ShaderLanguage lang);
 
   bool read(const ShaderFile &sfile, BamCacheRecord *record = nullptr);
+  bool load(const ShaderFile &sbody, BamCacheRecord *record = nullptr);
   bool do_read_source(std::string &into, const Filename &fn, BamCacheRecord *record);
-  bool r_preprocess_source(std::ostream &out, const Filename &fn,
-                           const Filename &source_dir,
+  bool do_load_source(std::string &into, const std::string &source, BamCacheRecord *record);
+  bool r_preprocess_include(std::ostream &out, const Filename &fn,
+                            const Filename &source_dir,
+                            std::set<Filename> &open_files,
+                            BamCacheRecord *record, int depth);
+  bool r_preprocess_source(std::ostream &out, std::istream &in,
+                           const Filename &fn, const Filename &full_fn,
                            std::set<Filename> &open_files,
-                           BamCacheRecord *record, int depth = 0);
+                           BamCacheRecord *record,
+                           int fileno = 0, int depth = 0);
 
   bool check_modified() const;