Support ptex mesh - step 5: improve the shader and rendering code (fa…

…cebookresearch#203)

* Support ptex mesh - step 5: improve the shader and rendering code

-) added gamma, saturation etc.;
-) refactored the PTexMeshShader a bit, moving function implementations to .cpp file;
-) cached the uniform positions;

* minor

* minor

* resolve Ci error

* rename gamma as gammaInverse in the shader

* add docstring, get rid of the inverse of gamma everywhere

* minor

src/esp/assets/PTexMeshData.cpp

@@ -53,10 +53,26 @@ float PTexMeshData::exposure() const {
   return exposure_;
 }
 
-void PTexMeshData::setExposure(const float& val) {
+void PTexMeshData::setExposure(float val) {
   exposure_ = val;
 }
 
+float PTexMeshData::gamma() const {
+  return gamma_;
+}
+
+void PTexMeshData::setGamma(float val) {
+  gamma_ = val;
+}
+
+float PTexMeshData::saturation() const {
+  return saturation_;
+}
+
+void PTexMeshData::setSaturation(float val) {
+  saturation_ = val;
+}
+
 const std::vector<PTexMeshData::MeshData>& PTexMeshData::meshes() const {
   return submeshes_;
 }

src/esp/assets/PTexMeshData.h

@@ -45,8 +45,6 @@ class PTexMeshData : public BaseMesh {
 
   // ==== geometry ====
   void load(const std::string& meshFile, const std::string& atlasFolder);
-  float exposure() const;
-  void setExposure(const float& val);
   uint32_t tileSize() const { return tileSize_; }
 
   const std::vector<MeshData>& meshes() const;
@@ -64,12 +62,31 @@ class PTexMeshData : public BaseMesh {
   virtual void uploadBuffersToGPU(bool forceReload = false) override;
   virtual Magnum::GL::Mesh* getMagnumGLMesh(int submeshID) override;
 
+  float exposure() const;
+  void setExposure(float val);
+
+  float gamma() const;
+  void setGamma(float val);
+
+  float saturation() const;
+  void setSaturation(float val);
+
  protected:
   void loadMeshData(const std::string& meshFile);
 
   float splitSize_ = 0.0f;
   uint32_t tileSize_ = 0;
-  float exposure_ = 1.0f;
+
+  // initial values are based on ReplicaSDK
+  //! @brief exposure, the amount of light per unit area reaching the image
+  float exposure_ = 0.025f;
+
+  //! @brief gamma, the exponent applied in the gamma correction
+  float gamma_ = 1.0f / 1.6969f;
+
+  //! @brief saturation, the intensity of a color
+  float saturation_ = 1.5f;
+
   std::string atlasFolder_;
   std::vector<MeshData> submeshes_;
 

src/esp/gfx/PTexMeshDrawable.cpp

@@ -21,12 +21,17 @@ PTexMeshDrawable::PTexMeshDrawable(
       adjFacesBufferTexture_(
           ptexMeshData.getRenderingBuffer(submeshID)->adjFacesBufferTexture),
       tileSize_(ptexMeshData.tileSize()),
-      exposure_(ptexMeshData.exposure()) {}
+      exposure_(ptexMeshData.exposure()),
+      gamma_(ptexMeshData.gamma()),
+      saturation_(ptexMeshData.saturation()) {}
 
 void PTexMeshDrawable::draw(const Magnum::Matrix4& transformationMatrix,
                             Magnum::SceneGraph::Camera3D& camera) {
   PTexMeshShader& ptexMeshShader = static_cast<PTexMeshShader&>(shader_);
-  ptexMeshShader.setPTexUniforms(atlasTexture_, tileSize_, exposure_)
+  ptexMeshShader.setExposure(exposure_)
+      .setGamma(gamma_)
+      .setSaturation(saturation_)
+      .setAtlasTextureSize(atlasTexture_, tileSize_)
       .bindAtlasTexture(atlasTexture_)
       .bindAdjFacesBufferTexture(adjFacesBufferTexture_)
       .setMVPMatrix(camera.projectionMatrix() * transformationMatrix);

src/esp/gfx/PTexMeshDrawable.h

@@ -32,6 +32,8 @@ class PTexMeshDrawable : public Drawable {
   Magnum::GL::BufferTexture& adjFacesBufferTexture_;
   uint32_t tileSize_;
   float exposure_;
+  float gamma_;
+  float saturation_;
 };
 
 }  // namespace gfx

src/esp/gfx/PTexMeshShader.cpp

@@ -2,12 +2,6 @@
 // This source code is licensed under the MIT license found in the
 // LICENSE file in the root directory of this source tree.
 
-#include "PTexMeshShader.h"
-
-#include <fcntl.h>
-#include <sys/mman.h>
-#include <iostream>
-
 #include <Corrade/Containers/Reference.h>
 #include <Corrade/Utility/Resource.h>
 #include <Magnum/GL/BufferTextureFormat.h>
@@ -17,6 +11,7 @@
 #include <Magnum/ImageView.h>
 #include <Magnum/PixelFormat.h>
 
+#include "PTexMeshShader.h"
 #include "esp/assets/PTexMeshData.h"
 #include "esp/core/esp.h"
 #include "esp/io/io.h"
@@ -70,6 +65,14 @@ PTexMeshShader::PTexMeshShader() {
   // TODO: disable the "meshAdjFaces" on Mac
   setUniform(uniformLocation("meshAdjFaces"),
              TextureBindingPointIndex::adjFaces);
+
+  // cache the uniform locations
+  MVPMatrixUniform_ = uniformLocation("MVP");
+  exposureUniform_ = uniformLocation("exposure");
+  gammaUniform_ = uniformLocation("gamma");
+  saturationUniform_ = uniformLocation("saturation");
+  tileSizeUniform_ = uniformLocation("tileSize");
+  widthInTilesUniform_ = uniformLocation("widthInTiles");
 }
 
 // Note: the texture binding points are explicitly specified above.
@@ -87,5 +90,36 @@ PTexMeshShader& PTexMeshShader::bindAdjFacesBufferTexture(
   return *this;
 }
 
+PTexMeshShader& PTexMeshShader::setMVPMatrix(const Magnum::Matrix4& matrix) {
+  setUniform(MVPMatrixUniform_, matrix);
+  return *this;
+}
+
+PTexMeshShader& PTexMeshShader::setExposure(float exposure) {
+  setUniform(exposureUniform_, exposure);
+  return *this;
+}
+PTexMeshShader& PTexMeshShader::setGamma(float gamma) {
+  setUniform(gammaUniform_, gamma);
+  return *this;
+}
+
+PTexMeshShader& PTexMeshShader::setSaturation(float saturation) {
+  setUniform(saturationUniform_, saturation);
+  return *this;
+}
+
+PTexMeshShader& PTexMeshShader::setAtlasTextureSize(
+    Magnum::GL::Texture2D& texture,
+    uint32_t tileSize) {
+  setUniform(tileSizeUniform_, (int)tileSize);
+
+  // get image width in given mip level 0
+  int mipLevel = 0;
+  const auto width = texture.imageSize(mipLevel).x();
+  setUniform(widthInTilesUniform_, int(width / tileSize));
+  return *this;
+}
+
 }  // namespace gfx
 }  // namespace esp

src/esp/gfx/PTexMeshShader.h

@@ -23,44 +23,64 @@ namespace gfx {
 
 class PTexMeshShader : public Magnum::GL::AbstractShaderProgram {
  public:
+  //! @brief vertex positions
   typedef Magnum::GL::Attribute<0, Magnum::Vector4> Position;
 
+  /**
+   * @brief Constructor
+   */
   explicit PTexMeshShader();
 
   // ======== texture binding ========
+  /**
+   * @brief Bind the atlas texture
+   * @return Reference to self (for method chaining)
+   */
   PTexMeshShader& bindAtlasTexture(Magnum::GL::Texture2D& texture);
+  /**
+   *  @brief Bind the buffer texture containing the adjacent faces
+   *  @return Reference to self (for method chaining)
+   */
   PTexMeshShader& bindAdjFacesBufferTexture(Magnum::GL::BufferTexture& texture);
 
   // ======== set uniforms ===========
-  PTexMeshShader& setMVPMatrix(const Magnum::Matrix4& matrix) {
-    setUniform(uniformLocation("MVP"), matrix);
-    return *this;
-  }
+  /**
+   *  @brief Set modelview and projection matrix to the uniform on GPU
+   *  @return Reference to self (for method chaining)
+   */
+  PTexMeshShader& setMVPMatrix(const Magnum::Matrix4& matrix);
+  /**
+   *  @brief Set expsure to the uniform on GPU
+   *  @return Reference to self (for method chaining)
+   */
+  PTexMeshShader& setExposure(float exposure);
+  /**
+   *  @brief Set gamma to the uniform on GPU
+   *  @return Reference to self (for method chaining)
+   */
+  PTexMeshShader& setGamma(float gamma);
+  /**
+   *  @brief Set saturation to the uniform on GPU
+   *  @return Reference to self (for method chaining)
+   */
+  PTexMeshShader& setSaturation(float saturation);
+  /**
+   *  @brief Set the tile size of the atlas texture
+   *  @return Reference to self (for method chaining)
+   */
+  PTexMeshShader& setAtlasTextureSize(Magnum::GL::Texture2D& texture,
+                                      uint32_t tileSize);
 
-  PTexMeshShader& setPTexUniforms(assets::PTexMeshData& ptexMeshData,
-                                  int submeshID,
-                                  uint32_t tileSize,
-                                  float exposure) {
-    setPTexUniforms(ptexMeshData.getRenderingBuffer(submeshID)->atlasTexture,
-                    tileSize, exposure);
-    return *this;
-  }
-
-  PTexMeshShader& setPTexUniforms(Magnum::GL::Texture2D& tex,
-                                  uint32_t tileSize,
-                                  float exposure) {
-    setUniform(uniformLocation("atlasTex"), 0);
-    setUniform(uniformLocation("tileSize"), static_cast<int>(tileSize));
-    // Image size in given mip level 0
-    {
-      int mipLevel = 0;
-      int widthEntry = 0;
-      const auto width = tex.imageSize(mipLevel)[widthEntry];
-      setUniform(uniformLocation("widthInTiles"), int(width / tileSize));
-    }
-    setUniform(uniformLocation("exposure"), exposure);
-    return *this;
-  }
+ protected:
+  // it hurts the performance to call glGetUniformLocation() every frame due to
+  // string operations.
+  // therefore, cache the locations in the constructor
+  int MVPMatrixUniform_;
+  int exposureUniform_;
+  int gammaUniform_;
+  int saturationUniform_;
+  int tileSizeUniform_;
+  int widthInTilesUniform_;
 };
 
 }  // namespace gfx

src/shaders/ptex-default-gl410.frag

@@ -155,15 +155,30 @@ vec4 textureAtlas(sampler2D tex, int faceID, vec2 p) {
       f.y);
 }
 
+void applySaturation(inout vec4 c, float saturation) {
+	float Pr = 0.299f;
+	float Pg = 0.587f;
+	float Pb = 0.114f;
+
+	float P = sqrt(c.r * c.r * Pr + c.g * c.g * Pg + c.b * c.b * Pb);
+
+	c.r = P + (c.r - P) * saturation;
+	c.g = P + (c.g - P) * saturation;
+	c.b = P + (c.b - P) * saturation;
+}
+
 layout(location = 0) out vec4 FragColor;
 uniform sampler2D atlasTex;
 
 uniform float exposure;
+uniform float gamma;
+uniform float saturation;
 
 in vec2 uv;
 
 void main() {
   vec4 c = textureAtlas(atlasTex, gl_PrimitiveID, uv * tileSize) * exposure;
-  // c = vec4(1.0f, 1.0f, 1.0f, 1.0f);
-  FragColor = vec4(c.xyz, 1.0f);
+	applySaturation(c, saturation);
+	c.rgb = pow(c.rgb, vec3(gamma));
+	FragColor = vec4(c.rgb, 1.0f);
 }