HDR pipeline: Use MRT to prefilter the environment map

Compositor/HDR/csm-pass.xml

@@ -6,7 +6,7 @@
   <effect-scheme>hdr-shadow</effect-scheme>
   <clear-mask>depth</clear-mask>
   <!-- Only render objects with the CAST_SHADOW bit flag -->
-  <cull-mask>0x000004</cull-mask>
+  <cull-mask>0x4</cull-mask>
   <render-at-night>false</render-at-night>
   <attachment>
     <buffer>sun-shadowmap-atlas</buffer>

Compositor/HDR/env-capture-pass.xml

@@ -3,10 +3,9 @@
 <PropertyList>
   <type>scene</type>
   <effect-scheme>hdr-envmap</effect-scheme>
-  <clear-mask>depth</clear-mask>
   <!-- Only render the skydome and terrain -->
   <!-- TODO: Explicitly select the LOD level -->
-  <cull-mask>0x801</cull-mask>
+  <cull-mask>0x800</cull-mask>
   <binding>
     <unit>11</unit>
     <buffer>sky-view</buffer>

Compositor/HDR/env-prefilter1-pass.xml

@@ -1,5 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-
-<PropertyList include="env-prefilter-pass.xml">
-  <effect>Effects/HDR/envmap-prefilter1</effect>
-</PropertyList>

Compositor/HDR/env-prefilter2-pass.xml

@@ -1,5 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-
-<PropertyList include="env-prefilter-pass.xml">
-  <effect>Effects/HDR/envmap-prefilter2</effect>
-</PropertyList>

Compositor/HDR/env-prefilter3-pass.xml

@@ -1,5 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-
-<PropertyList include="env-prefilter-pass.xml">
-  <effect>Effects/HDR/envmap-prefilter3</effect>
-</PropertyList>

Compositor/HDR/env-prefilter4-pass.xml

@@ -1,5 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-
-<PropertyList include="env-prefilter-pass.xml">
-  <effect>Effects/HDR/envmap-prefilter4</effect>
-</PropertyList>

Compositor/HDR/hdr.xml

@@ -315,244 +315,164 @@
       Diffuse lighting is approximated by using the highest mipmap level
       (roughness=1).
   -->
-  <pass include="env-prefilter1-pass.xml">
+  <pass include="env-prefilter-pass.xml">
-    <name>env-prefilter1-0</name>
+    <name>env-prefilter1</name>
-    <cubemap-face>0</cubemap-face>
+    <effect>Effects/HDR/envmap-prefilter1</effect>
     <attachment>
       <component>color0</component>
       <buffer>envmap</buffer>
       <face>0</face>
       <level>1</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter1-pass.xml">
-    <name>env-prefilter1-1</name>
-    <cubemap-face>1</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color1</component>
       <buffer>envmap</buffer>
       <face>1</face>
       <level>1</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter1-pass.xml">
-    <name>env-prefilter1-2</name>
-    <cubemap-face>2</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color2</component>
       <buffer>envmap</buffer>
       <face>2</face>
       <level>1</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter1-pass.xml">
-    <name>env-prefilter1-3</name>
-    <cubemap-face>3</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color3</component>
       <buffer>envmap</buffer>
       <face>3</face>
       <level>1</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter1-pass.xml">
-    <name>env-prefilter1-4</name>
-    <cubemap-face>4</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color4</component>
       <buffer>envmap</buffer>
       <face>4</face>
       <level>1</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter1-pass.xml">
-    <name>env-prefilter1-5</name>
-    <cubemap-face>5</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color5</component>
       <buffer>envmap</buffer>
       <face>5</face>
       <level>1</level>
     </attachment>
   </pass>
 
-  <pass include="env-prefilter2-pass.xml">
+  <pass include="env-prefilter-pass.xml">
-    <name>env-prefilter2-0</name>
+    <name>env-prefilter2</name>
-    <cubemap-face>0</cubemap-face>
+    <effect>Effects/HDR/envmap-prefilter2</effect>
     <attachment>
       <component>color0</component>
       <buffer>envmap</buffer>
       <face>0</face>
       <level>2</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter2-pass.xml">
-    <name>env-prefilter2-1</name>
-    <cubemap-face>1</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color1</component>
       <buffer>envmap</buffer>
       <face>1</face>
       <level>2</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter2-pass.xml">
-    <name>env-prefilter2-2</name>
-    <cubemap-face>2</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color2</component>
       <buffer>envmap</buffer>
       <face>2</face>
       <level>2</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter2-pass.xml">
-    <name>env-prefilter2-3</name>
-    <cubemap-face>3</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color3</component>
       <buffer>envmap</buffer>
       <face>3</face>
       <level>2</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter2-pass.xml">
-    <name>env-prefilter2-4</name>
-    <cubemap-face>4</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color4</component>
       <buffer>envmap</buffer>
       <face>4</face>
       <level>2</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter2-pass.xml">
-    <name>env-prefilter2-5</name>
-    <cubemap-face>5</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color5</component>
       <buffer>envmap</buffer>
       <face>5</face>
       <level>2</level>
     </attachment>
   </pass>
 
-  <pass include="env-prefilter3-pass.xml">
+  <pass include="env-prefilter-pass.xml">
-    <name>env-prefilter3-0</name>
+    <name>env-prefilter3</name>
-    <cubemap-face>0</cubemap-face>
+    <effect>Effects/HDR/envmap-prefilter3</effect>
     <attachment>
       <component>color0</component>
       <buffer>envmap</buffer>
       <face>0</face>
       <level>3</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter3-pass.xml">
-    <name>env-prefilter3-1</name>
-    <cubemap-face>1</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color1</component>
       <buffer>envmap</buffer>
       <face>1</face>
       <level>3</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter3-pass.xml">
-    <name>env-prefilter3-2</name>
-    <cubemap-face>2</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color2</component>
       <buffer>envmap</buffer>
       <face>2</face>
       <level>3</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter3-pass.xml">
-    <name>env-prefilter3-3</name>
-    <cubemap-face>3</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color3</component>
       <buffer>envmap</buffer>
       <face>3</face>
       <level>3</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter3-pass.xml">
-    <name>env-prefilter3-4</name>
-    <cubemap-face>4</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color4</component>
       <buffer>envmap</buffer>
       <face>4</face>
       <level>3</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter3-pass.xml">
-    <name>env-prefilter3-5</name>
-    <cubemap-face>5</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color5</component>
       <buffer>envmap</buffer>
       <face>5</face>
       <level>3</level>
     </attachment>
   </pass>
 
-  <pass include="env-prefilter4-pass.xml">
+  <pass include="env-prefilter-pass.xml">
-    <name>env-prefilter4-0</name>
+    <name>env-prefilter4</name>
-    <cubemap-face>0</cubemap-face>
+    <effect>Effects/HDR/envmap-prefilter4</effect>
     <attachment>
       <component>color0</component>
       <buffer>envmap</buffer>
       <face>0</face>
       <level>4</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter4-pass.xml">
-    <name>env-prefilter4-1</name>
-    <cubemap-face>1</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color1</component>
       <buffer>envmap</buffer>
       <face>1</face>
       <level>4</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter4-pass.xml">
-    <name>env-prefilter4-2</name>
-    <cubemap-face>2</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color2</component>
       <buffer>envmap</buffer>
       <face>2</face>
       <level>4</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter4-pass.xml">
-    <name>env-prefilter4-3</name>
-    <cubemap-face>3</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color3</component>
       <buffer>envmap</buffer>
       <face>3</face>
       <level>4</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter4-pass.xml">
-    <name>env-prefilter4-4</name>
-    <cubemap-face>4</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color4</component>
       <buffer>envmap</buffer>
       <face>4</face>
       <level>4</level>
     </attachment>
-  </pass>
-  <pass include="env-prefilter4-pass.xml">
-    <name>env-prefilter4-5</name>
-    <cubemap-face>5</cubemap-face>
     <attachment>
-      <component>color0</component>
+      <component>color5</component>
       <buffer>envmap</buffer>
       <face>5</face>
       <level>4</level>

Effects/HDR/envmap-prefilter.eff

@@ -7,7 +7,7 @@
   <technique n="1">
     <pass>
       <program>
-        <vertex-shader>Shaders/HDR/trivial-cubemap.vert</vertex-shader>
+        <vertex-shader>Shaders/HDR/envmap-prefilter.vert</vertex-shader>
         <fragment-shader>Shaders/HDR/envmap-prefilter.frag</fragment-shader>
       </program>
       <uniform>

Shaders/HDR/envmap-prefilter.frag

@@ -1,8 +1,18 @@
 #version 330 core
 
-out vec3 fragColor;
+layout(location = 0) out vec3 fragColor0;
+layout(location = 1) out vec3 fragColor1;
+layout(location = 2) out vec3 fragColor2;
+layout(location = 3) out vec3 fragColor3;
+layout(location = 4) out vec3 fragColor4;
+layout(location = 5) out vec3 fragColor5;
 
-in vec3 cubemapCoord;
+in vec3 cubemapCoord0;
+in vec3 cubemapCoord1;
+in vec3 cubemapCoord2;
+in vec3 cubemapCoord3;
+in vec3 cubemapCoord4;
+in vec3 cubemapCoord5;
 
 uniform samplerCube envmap;
 uniform float roughness;
@@ -10,6 +20,7 @@ uniform float roughness;
 uniform int fg_CubemapFace;
 
 const float PI = 3.14159265359;
+const uint NUM_SAMPLES = 64u;
 
 float RadicalInverse_VdC(uint bits)
 {
@@ -47,14 +58,13 @@ vec3 ImportanceSampleGGX(vec2 Xi, vec3 n, float r)
     return normalize(sampleVec);
 }
 
-void main()
+vec3 prefilter(vec3 n)
 {
-    vec3 n = normalize(cubemapCoord);
+    n = normalize(n);
     vec3 v = n; // n = v simplification
 
     vec3 prefilteredColor = vec3(0.0);
     float totalWeight = 0.0;
-    const uint NUM_SAMPLES = 1024u;
 
     for (uint i = 0u; i < NUM_SAMPLES; ++i) {
         vec2 Xi = Hammersley(i, NUM_SAMPLES);
@@ -69,6 +79,15 @@ void main()
     }
 
     prefilteredColor /= totalWeight;
-
+    return prefilteredColor;
-    fragColor = prefilteredColor;
+}
+
+void main()
+{
+    fragColor0 = prefilter(cubemapCoord0);
+    fragColor1 = prefilter(cubemapCoord1);
+    fragColor2 = prefilter(cubemapCoord2);
+    fragColor3 = prefilter(cubemapCoord3);
+    fragColor4 = prefilter(cubemapCoord4);
+    fragColor5 = prefilter(cubemapCoord5);
 }

Shaders/HDR/envmap-prefilter.vert

@@ -0,0 +1,29 @@
+#version 330 core
+
+layout(location = 0) in vec4 pos;
+layout(location = 3) in vec4 multiTexCoord0;
+
+out vec3 cubemapCoord0;
+out vec3 cubemapCoord1;
+out vec3 cubemapCoord2;
+out vec3 cubemapCoord3;
+out vec3 cubemapCoord4;
+out vec3 cubemapCoord5;
+
+uniform mat4 osg_ModelViewProjectionMatrix;
+
+void main()
+{
+    gl_Position = osg_ModelViewProjectionMatrix * pos;
+    vec2 texCoord = multiTexCoord0.xy * 2.0 - 1.0;
+    // Map the quad texture coordinates to a direction vector to sample
+    // the cubemap. This assumes that we are using the weird left-handed
+    // orientations given by the OpenGL spec.
+    // See https://www.khronos.org/opengl/wiki/Cubemap_Texture#Upload_and_orientation
+    cubemapCoord0 = vec3(1.0, -texCoord.y, -texCoord.x);
+    cubemapCoord1 = vec3(-1.0, -texCoord.y, texCoord.x);
+    cubemapCoord2 = vec3(texCoord.x, 1.0, texCoord.y);
+    cubemapCoord3 = vec3(texCoord.x, -1.0, -texCoord.y);
+    cubemapCoord4 = vec3(texCoord.x, -texCoord.y, 1.0);
+    cubemapCoord5 = vec3(-texCoord.x, -texCoord.y, -1.0);
+}