1

I am trying to get the depth values of each pixel in the canvas element. Is there a way to find these depth values using WebGL and Three.js?

What I majorly want is that for eg. in the image below, the red background should have 0 as the depth value whereas the 3D model should have the depth values based on the distance from the camera.

Using the X,Y coordinates of the canvas, is there a method to access the depth values?

Image of my current scene

[Edit 1]: Adding more information

I pick three random points as shown below, then I ask the user to input the depth values for each of these points. Once the input is received from the user, I will compute the difference between the depth values in three.js and the values inputted from the user.

Basically, I would require a 2D array of the canvas size where each pixel corresponds to an array value. This 2D array must contain the value 0 if the pixel is a red background, or contain the depth value if the pixel contains the 3D model.

enter image description here

7
  • Take a look at this example: threejs.org/examples/?q=depth#webgl_depth_texture Commented Nov 18, 2019 at 12:18
  • Your question really needs more context. What do you want these depth values for? Are you going to use them to draw something else? The reason I ask is that the answer will change significantly depending on what you need them for. As an exmaple, drawing shadows requires depth values so clearly it's easy to get depth values but JavaScript never sees those values. They are rendering on the GPU to a texture and then the texture with the depth values is used again on the GPU to do shadow calculations. Commented Nov 18, 2019 at 13:00
  • @prisoner849 - thanks a lot for the example. It looks complicated to me, I only need access to the depth values of the canvas once I draw the scene. Commented Nov 18, 2019 at 13:28
  • So just curious, if you only need a few points like 3 then just using the RayCaster will reutrn the depth. Example: jsfiddle.net/greggman/nh5pv4y1 Commented Nov 18, 2019 at 16:56
  • 1
    You can get random points with raycasting. Just pass in the coordinate of the pixel you want and cast the ray, repeat. jsfiddle.net/greggman/syoq2xf5 Commented Nov 19, 2019 at 13:22

1 Answer 1

4

Two ways come to mind.

One you can just use RayCaster

body {
  margin: 0;
}
#c {
  width: 100vw;
  height: 100vh;
  display: block;
}
.info {
  position: absolute;
  left: 1em;
  top: 1em;
  padding: 1em;
  background: rgba(0, 0, 0, 0.7);
  color: white;
  font-size: xx-small;
}
.info::after{
  content: '';
  position: absolute;
  border: 10px solid transparent;
  border-top: 10px solid rgba(0, 0, 0, 0.7);
  top: 0;
  left: -10px;
}
<canvas id="c"></canvas>
<script type="module">
// Three.js - Picking - RayCaster
// from https://threejsfundamentals.org/threejs/threejs-picking-raycaster.html

import * as THREE from 'https://threejsfundamentals.org/threejs/resources/threejs/r110/build/three.module.js';

function main() {
  const canvas = document.querySelector('#c');
  const renderer = new THREE.WebGLRenderer({canvas});

  const fov = 60;
  const aspect = 2;  // the canvas default
  const near = 0.1;
  const far = 200;
  const camera = new THREE.PerspectiveCamera(fov, aspect, near, far);
  camera.position.z = 30;

  const points = [
    [170, 20],
    [400, 50],
    [225, 120],
  ].map((point) => {
    const infoElem = document.createElement('pre');
    document.body.appendChild(infoElem);
    infoElem.className = "info";
    infoElem.style.left = `${point[0] + 10}px`;
    infoElem.style.top = `${point[1]}px`;
    return {
      point,
      infoElem,
    };
  });
  
  const scene = new THREE.Scene();
  scene.background = new THREE.Color('white');

  // put the camera on a pole (parent it to an object)
  // so we can spin the pole to move the camera around the scene
  const cameraPole = new THREE.Object3D();
  scene.add(cameraPole);
  cameraPole.add(camera);

  {
    const color = 0xFFFFFF;
    const intensity = 1;
    const light = new THREE.DirectionalLight(color, intensity);
    light.position.set(-1, 2, 4);
    camera.add(light);
  }

  const boxWidth = 1;
  const boxHeight = 1;
  const boxDepth = 1;
  const geometry = new THREE.BoxGeometry(boxWidth, boxHeight, boxDepth);

  function rand(min, max) {
    if (max === undefined) {
      max = min;
      min = 0;
    }
    return min + (max - min) * Math.random();
  }

  function randomColor() {
    return `hsl(${rand(360) | 0}, ${rand(50, 100) | 0}%, 50%)`;
  }

  const numObjects = 100;
  for (let i = 0; i < numObjects; ++i) {
    const material = new THREE.MeshPhongMaterial({
      color: randomColor(),
    });

    const cube = new THREE.Mesh(geometry, material);
    scene.add(cube);

    cube.position.set(rand(-20, 20), rand(-20, 20), rand(-20, 20));
    cube.rotation.set(rand(Math.PI), rand(Math.PI), 0);
    cube.scale.set(rand(3, 6), rand(3, 6), rand(3, 6));
  }

  function resizeRendererToDisplaySize(renderer) {
    const canvas = renderer.domElement;
    const width = canvas.clientWidth;
    const height = canvas.clientHeight;
    const needResize = canvas.width !== width || canvas.height !== height;
    if (needResize) {
      renderer.setSize(width, height, false);
    }
    return needResize;
  }

  const raycaster = new THREE.Raycaster();

  function render(time) {
    time *= 0.001;  // convert to seconds;

    if (resizeRendererToDisplaySize(renderer)) {
      const canvas = renderer.domElement;
      camera.aspect = canvas.clientWidth / canvas.clientHeight;
      camera.updateProjectionMatrix();
    }

    cameraPole.rotation.y = time * .1;

    for (const {point, infoElem} of points) {
      const pickPosition = {
        x: (point[0] / canvas.clientWidth ) *  2 - 1,
		    y: (point[1] / canvas.clientHeight) * -2 + 1,  // note we flip Y
      };
      raycaster.setFromCamera(pickPosition, camera);
      const intersectedObjects = raycaster.intersectObjects(scene.children);
      if (intersectedObjects.length) {
        // pick the first object. It's the closest one
        const intersection = intersectedObjects[0];
        infoElem.textContent = `position : ${point[0]}, ${point[1]}
distance : ${intersection.distance.toFixed(2)}
z depth  : ${((intersection.distance - near) / (far - near)).toFixed(3)}
local pos: ${intersection.point.x.toFixed(2)}, ${intersection.point.y.toFixed(2)}, ${intersection.point.z.toFixed(2)}
local uv : ${intersection.uv.x.toFixed(2)}, ${intersection.uv.y.toFixed(2)}`;
      } else {
        infoElem.textContent = `position : ${point[0]}, ${point[1]}`;
      }
    }
    renderer.render(scene, camera);

    requestAnimationFrame(render);
  }
  requestAnimationFrame(render);
}

main();
</script>

The other way is to do as you mentioned and read the depth buffer. Unfortunately there is no direct way to read the depth buffer.

To read the depth values you need 2 render targets. You'd render to the first target. That gives you both a color texture with the rendered image and a depth texture with the depth values. You can't read a depth texture directly but you can draw it to another color texture and then read the color texture. Finally you can draw the first color texture to the cavnas.

body {
  margin: 0;
}
#c {
  width: 100vw;
  height: 100vh;
  display: block;
}
.info {
  position: absolute;
  left: 1em;
  top: 1em;
  padding: 1em;
  background: rgba(0, 0, 0, 0.7);
  color: white;
  font-size: xx-small;
}
.info::after{
  content: '';
  position: absolute;
  border: 10px solid transparent;
  border-top: 10px solid rgba(0, 0, 0, 0.7);
  top: 0;
  left: -10px;
}
<canvas id="c"></canvas>
<script type="module">
import * as THREE from 'https://threejsfundamentals.org/threejs/resources/threejs/r110/build/three.module.js';

function main() {
  const canvas = document.querySelector('#c');
  const renderer = new THREE.WebGLRenderer({canvas});
  
  const points = [
     [170, 20],
     [400, 50],
     [225, 120],
  ].map((point) => {
    const infoElem = document.createElement('pre');
    document.body.appendChild(infoElem);
    infoElem.className = "info";
    infoElem.style.left = `${point[0] + 10}px`;
    infoElem.style.top = `${point[1]}px`;
    return {
      point,
      infoElem,
    };
  });
  
  const renderTarget = new THREE.WebGLRenderTarget(1, 1);
  renderTarget.depthTexture = new THREE.DepthTexture();
  const depthRenderTarget = new THREE.WebGLRenderTarget(1, 1, {
    depthBuffer: false,
    stenciBuffer: false,
  });

  const rtFov = 60;
  const rtAspect = 1;
  const rtNear = 0.1;
  const rtFar = 200;
  const rtCamera = new THREE.PerspectiveCamera(rtFov, rtAspect, rtNear, rtFar);
  rtCamera.position.z = 30;

  const rtScene = new THREE.Scene();
  rtScene.background = new THREE.Color('white');

  // put the camera on a pole (parent it to an object)
  // so we can spin the pole to move the camera around the scene
  const cameraPole = new THREE.Object3D();
  rtScene.add(cameraPole);
  cameraPole.add(rtCamera);

  {
    const color = 0xFFFFFF;
    const intensity = 1;
    const light = new THREE.DirectionalLight(color, intensity);
    light.position.set(-1, 2, 4);
    rtCamera.add(light);
  }

  const boxWidth = 1;
  const boxHeight = 1;
  const boxDepth = 1;
  const geometry = new THREE.BoxGeometry(boxWidth, boxHeight, boxDepth);

  function rand(min, max) {
    if (max === undefined) {
      max = min;
      min = 0;
    }
    return min + (max - min) * Math.random();
  }

  function randomColor() {
    return `hsl(${rand(360) | 0}, ${rand(50, 100) | 0}%, 50%)`;
  }

  const numObjects = 100;
  for (let i = 0; i < numObjects; ++i) {
    const material = new THREE.MeshPhongMaterial({
      color: randomColor(),
    });

    const cube = new THREE.Mesh(geometry, material);
    rtScene.add(cube);

    cube.position.set(rand(-20, 20), rand(-20, 20), rand(-20, 20));
    cube.rotation.set(rand(Math.PI), rand(Math.PI), 0);
    cube.scale.set(rand(3, 6), rand(3, 6), rand(3, 6));
  }


  const camera = new THREE.OrthographicCamera(-1, 1, 1, -1, -1, 1);
  const scene = new THREE.Scene();
  camera.position.z = 1;

  const sceneMaterial = new THREE.MeshBasicMaterial({
    map: renderTarget.texture,
  });
  const planeGeo = new THREE.PlaneBufferGeometry(2, 2);
  const plane = new THREE.Mesh(planeGeo, sceneMaterial);
  scene.add(plane);
  
  const depthScene = new THREE.Scene();
  const depthMaterial = new THREE.MeshBasicMaterial({
    map: renderTarget.depthTexture,
  });
  const depthPlane = new THREE.Mesh(planeGeo, depthMaterial);
  depthScene.add(depthPlane);
  

  function resizeRendererToDisplaySize(renderer) {
    const canvas = renderer.domElement;
    const width = canvas.clientWidth;
    const height = canvas.clientHeight;
    const needResize = canvas.width !== width || canvas.height !== height;
    if (needResize) {
      renderer.setSize(width, height, false);
    }
    return needResize;
  }

  let depthValues = new Uint8Array(0);
  function render(time) {
    time *= 0.001;

    if (resizeRendererToDisplaySize(renderer)) {
      const canvas = renderer.domElement;
      renderTarget.setSize(canvas.width, canvas.height);
      depthRenderTarget.setSize(canvas.width, canvas.height);
      rtCamera.aspect = canvas.clientWidth / canvas.clientHeight;
      rtCamera.updateProjectionMatrix();
    }

    cameraPole.rotation.y = time * .1;

    // draw render target scene to render target
    renderer.setRenderTarget(renderTarget);
    renderer.render(rtScene, rtCamera);
    renderer.setRenderTarget(null);
    
    // render the depth texture to another render target
    renderer.setRenderTarget(depthRenderTarget);
    renderer.render(depthScene, camera);
    renderer.setRenderTarget(null);

    {
      const {width, height} = depthRenderTarget;
      const spaceNeeded = width * height * 4;
      if (depthValues.length !== spaceNeeded) {
        depthValues = new Uint8Array(spaceNeeded);
      }
      renderer.readRenderTargetPixels(
          depthRenderTarget,
          0,
          0,
          depthRenderTarget.width,
          depthRenderTarget.height,
          depthValues);

      for (const {point, infoElem} of points) {
         const offset = ((height - point[1] - 1) * width + point[0]) * 4;
         infoElem.textContent = `position : ${point[0]}, ${point[1]}
z depth  : ${(depthValues[offset] / 255).toFixed(3)}`;
      }    
    }
    
    // render the color texture to the canvas
    renderer.render(scene, camera);

    requestAnimationFrame(render);
  }

  requestAnimationFrame(render);
}

main();
</script>

The problem is you can only read UNSIGNED_BYTE values from the texture so your depth values only go from 0 to 255 which is not really enough resolution to do much.

To solve that issue you have to encode the depth values across channels when drawing the depth texture to the 2nd render target which means you need to make your own shader. three.js has some shader snippets for packing the values so hacking a shader using ideas from this article we can get better depth values.

body {
  margin: 0;
}
#c {
  width: 100vw;
  height: 100vh;
  display: block;
}
.info {
  position: absolute;
  left: 1em;
  top: 1em;
  padding: 1em;
  background: rgba(0, 0, 0, 0.7);
  color: white;
  font-size: xx-small;
}
.info::after{
  content: '';
  position: absolute;
  border: 10px solid transparent;
  border-top: 10px solid rgba(0, 0, 0, 0.7);
  top: 0;
  left: -10px;
}
<canvas id="c"></canvas>
<script type="module">
import * as THREE from 'https://threejsfundamentals.org/threejs/resources/threejs/r110/build/three.module.js';

function main() {
  const canvas = document.querySelector('#c');
  const renderer = new THREE.WebGLRenderer({canvas});
  
  const points = [
     [170, 20],
     [400, 50],
     [225, 120],
  ].map((point) => {
    const infoElem = document.createElement('pre');
    document.body.appendChild(infoElem);
    infoElem.className = "info";
    infoElem.style.left = `${point[0] + 10}px`;
    infoElem.style.top = `${point[1]}px`;
    return {
      point,
      infoElem,
    };
  });
  
  const renderTarget = new THREE.WebGLRenderTarget(1, 1);
  renderTarget.depthTexture = new THREE.DepthTexture();
  const depthRenderTarget = new THREE.WebGLRenderTarget(1, 1, {
    depthBuffer: false,
    stenciBuffer: false,
  });

  const rtFov = 60;
  const rtAspect = 1;
  const rtNear = 0.1;
  const rtFar = 200;
  const rtCamera = new THREE.PerspectiveCamera(rtFov, rtAspect, rtNear, rtFar);
  rtCamera.position.z = 30;

  const rtScene = new THREE.Scene();
  rtScene.background = new THREE.Color('white');

  // put the camera on a pole (parent it to an object)
  // so we can spin the pole to move the camera around the scene
  const cameraPole = new THREE.Object3D();
  rtScene.add(cameraPole);
  cameraPole.add(rtCamera);

  {
    const color = 0xFFFFFF;
    const intensity = 1;
    const light = new THREE.DirectionalLight(color, intensity);
    light.position.set(-1, 2, 4);
    rtCamera.add(light);
  }

  const boxWidth = 1;
  const boxHeight = 1;
  const boxDepth = 1;
  const geometry = new THREE.BoxGeometry(boxWidth, boxHeight, boxDepth);

  function rand(min, max) {
    if (max === undefined) {
      max = min;
      min = 0;
    }
    return min + (max - min) * Math.random();
  }

  function randomColor() {
    return `hsl(${rand(360) | 0}, ${rand(50, 100) | 0}%, 50%)`;
  }

  const numObjects = 100;
  for (let i = 0; i < numObjects; ++i) {
    const material = new THREE.MeshPhongMaterial({
      color: randomColor(),
    });

    const cube = new THREE.Mesh(geometry, material);
    rtScene.add(cube);

    cube.position.set(rand(-20, 20), rand(-20, 20), rand(-20, 20));
    cube.rotation.set(rand(Math.PI), rand(Math.PI), 0);
    cube.scale.set(rand(3, 6), rand(3, 6), rand(3, 6));
  }


  const camera = new THREE.OrthographicCamera(-1, 1, 1, -1, -1, 1);
  const scene = new THREE.Scene();
  camera.position.z = 1;

  const sceneMaterial = new THREE.MeshBasicMaterial({
    map: renderTarget.texture,
  });
  const planeGeo = new THREE.PlaneBufferGeometry(2, 2);
  const plane = new THREE.Mesh(planeGeo, sceneMaterial);
  scene.add(plane);
  
  const depthScene = new THREE.Scene();
  const depthMaterial = new THREE.MeshBasicMaterial({
    map: renderTarget.depthTexture,
  });
  depthMaterial.onBeforeCompile = function(shader) {
    // the <packing> GLSL chunk from three.js has the packDeathToRGBA function.
    // then at the end of the shader the default MaterialBasicShader has
    // already read from the material's `map` texture (the depthTexture)
    // which has depth in 'r' and assigned it to gl_FragColor
    shader.fragmentShader = shader.fragmentShader.replace(
        '#include <common>',
        '#include <common>\n#include <packing>',
    ).replace(
        '#include <fog_fragment>',
        'gl_FragColor = packDepthToRGBA( gl_FragColor.r );',
    );
  };  

  const depthPlane = new THREE.Mesh(planeGeo, depthMaterial);
  depthScene.add(depthPlane);

  function resizeRendererToDisplaySize(renderer) {
    const canvas = renderer.domElement;
    const width = canvas.clientWidth;
    const height = canvas.clientHeight;
    const needResize = canvas.width !== width || canvas.height !== height;
    if (needResize) {
      renderer.setSize(width, height, false);
    }
    return needResize;
  }

  let depthValues = new Uint8Array(0);
  function render(time) {
    time *= 0.001;

    if (resizeRendererToDisplaySize(renderer)) {
      const canvas = renderer.domElement;
      renderTarget.setSize(canvas.width, canvas.height);
      depthRenderTarget.setSize(canvas.width, canvas.height);
      rtCamera.aspect = canvas.clientWidth / canvas.clientHeight;
      rtCamera.updateProjectionMatrix();
    }

    cameraPole.rotation.y = time * .1;

    // draw render target scene to render target
    renderer.setRenderTarget(renderTarget);
    renderer.render(rtScene, rtCamera);
    renderer.setRenderTarget(null);
    
    // render the depth texture to another render target
    renderer.setRenderTarget(depthRenderTarget);
    renderer.render(depthScene, camera);
    renderer.setRenderTarget(null);

    {
      const {width, height} = depthRenderTarget;
      const spaceNeeded = width * height * 4;
      if (depthValues.length !== spaceNeeded) {
        depthValues = new Uint8Array(spaceNeeded);
      }
      renderer.readRenderTargetPixels(
          depthRenderTarget,
          0,
          0,
          depthRenderTarget.width,
          depthRenderTarget.height,
          depthValues);
          
      for (const {point, infoElem} of points) {
         const offset = ((height - point[1] - 1) * width + point[0]) * 4;
         const depth = depthValues[offset    ] * ((255 / 256) / (256 * 256 * 256)) +
                       depthValues[offset + 1] * ((255 / 256) / (256 * 256)) +
                       depthValues[offset + 2] * ((255 / 256) / 256);

         
         infoElem.textContent = `position : ${point[0]}, ${point[1]}
z depth  : ${depth.toFixed(3)}`;
      }    
    }
    
    // render the color texture to the canvas
    renderer.render(scene, camera);

    requestAnimationFrame(render);
  }

  requestAnimationFrame(render);
}

main();
</script>

Note depthTexture uses a webgl extension which is an optional feature not found on all devices

To work around that would require drawing the scene twice. Once with your normal materials and then again to a color render target using the MeshDepthMaterial.

Sign up to request clarification or add additional context in comments.

1 Comment

thank you very much. Your method was really helpful to solve the problem :)

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.