How to pass off heavy JavaScript math operations to GPU with GPU.js

Question

Background
I've built a little web based application that pops up windows to display your webcam(s). I wanted to add the ability to chroma key your feed and have been successful in getting several different algorithms working. The best algorithm I have found however is very resource intensive for JavaScript; single threaded application.

Question
Is there a way to offload the intensive math operations to the GPU? I've tried getting GPU.js to work but I keep getting all kinds of errors. Here is the functions I would like to have the GPU run:

let dE76 = function(a, b, c, d, e, f) {
    return Math.sqrt( pow(d - a, 2) + pow(e - b, 2) + pow(f - c, 2) );
};


let rgbToLab = function(r, g, b) {
    
    let x, y, z;

    r = r / 255;
    g = g / 255;
    b = b / 255;

    r = (r > 0.04045) ? Math.pow((r + 0.055) / 1.055, 2.4) : r / 12.92;
    g = (g > 0.04045) ? Math.pow((g + 0.055) / 1.055, 2.4) : g / 12.92;
    b = (b > 0.04045) ? Math.pow((b + 0.055) / 1.055, 2.4) : b / 12.92;

    x = (r * 0.4124 + g * 0.3576 + b * 0.1805) / 0.95047;
    y = (r * 0.2126 + g * 0.7152 + b * 0.0722) / 1.00000;
    z = (r * 0.0193 + g * 0.1192 + b * 0.9505) / 1.08883;

    x = (x > 0.008856) ? Math.pow(x, 1/3) : (7.787 * x) + 16/116;
    y = (y > 0.008856) ? Math.pow(y, 1/3) : (7.787 * y) + 16/116;
    z = (z > 0.008856) ? Math.pow(z, 1/3) : (7.787 * z) + 16/116;

    return [ (116 * y) - 16, 500 * (x - y), 200 * (y - z) ];
};

What happens here is I send in an RGB value to rgbToLab which gives back the LAB value that can be compared to an already stored LAB value for my green screen with dE76. Then in my app we check the dE76 value to a threashold, say 25, and if the value is less than this I turn that pixel opacity to 0 in the video feed.

GPU.js Attempt
Here is my latest GUI.js attempt:

// Try to combine the 2 functions into a single kernel function for GPU.js
let tmp = gpu.createKernel( function( r, g, b, lab ) {

  let x, y, z;

  r = r / 255;
  g = g / 255;
  b = b / 255;

  r = (r > 0.04045) ? Math.pow((r + 0.055) / 1.055, 2.4) : r / 12.92;
  g = (g > 0.04045) ? Math.pow((g + 0.055) / 1.055, 2.4) : g / 12.92;
  b = (b > 0.04045) ? Math.pow((b + 0.055) / 1.055, 2.4) : b / 12.92;

  x = (r * 0.4124 + g * 0.3576 + b * 0.1805) / 0.95047;
  y = (r * 0.2126 + g * 0.7152 + b * 0.0722) / 1.00000;
  z = (r * 0.0193 + g * 0.1192 + b * 0.9505) / 1.08883;

  x = (x > 0.008856) ? Math.pow(x, 1/3) : (7.787 * x) + 16/116;
  y = (y > 0.008856) ? Math.pow(y, 1/3) : (7.787 * y) + 16/116;
  z = (z > 0.008856) ? Math.pow(z, 1/3) : (7.787 * z) + 16/116;

  let clab = [ (116 * y) - 16, 500 * (x - y), 200 * (y - z) ];
  
  let d = pow(lab[0] - clab[0], 2) + pow(lab[1] - clab[1], 2) + pow(lab[2] - clab[2], 2);
  
  return Math.sqrt( d );

} ).setOutput( [256] );

// ...

// Call the function above.
let d = tmp( r, g, b, chromaColors[c].lab );

// If the delta (d) is lower than my tolerance level set pixel opacity to 0.
if( d < tolerance ){
    frame.data[ i * 4 + 3 ] = 0;
}

ERRORS:
Here are a list of errors I get trying to use GPU.js when I call my tmp function. 1) is for the code I provided above. 2) is for erasing all the code in tmp and adding only an empty return 3) is if I try and add the functions inside the tmp function; a valid JavaScript thing but not C or kernel code.

Uncaught Error: Identifier is not defined
Uncaught Error: Error compiling fragment shader: ERROR: 0:463: ';' : syntax error
Uncaught Error: Unhandled type FunctionExpression in getDependencies

You might want to use fragment shaders with webgl. Their is also this: github.com/turbo/js — user2258152
– user2258152, Commented Sep 24, 2020 at 23:09
I tried turbojs but I kept getting kernel code errors. I might take the time to combine these two functions and convert them to C and give turbojs another shot. — Blizzardengle
– Blizzardengle, Commented Sep 24, 2020 at 23:18
I don't know helpful they will be in this case but I added some. — Blizzardengle
– Blizzardengle, Commented Sep 25, 2020 at 0:46

user2258152 · Accepted Answer · 2020-09-25 01:10:47Z

1

Some typos

pow should be Math.pow()

and

let x, y, z should be declare on there own

let x = 0
let y = 0
let z = 0

You cannot assign value to parameter variable. They become uniform.

Full working script

const { GPU } = require('gpu.js')
const gpu = new GPU()

const tmp = gpu.createKernel(function (r, g, b, lab) {
  let x = 0
  let y = 0
  let z = 0

  let r1 = r / 255
  let g1 = g / 255
  let b1 = b / 255

  r1 = (r1 > 0.04045) ? Math.pow((r1 + 0.055) / 1.055, 2.4) : r1 / 12.92
  g1 = (g1 > 0.04045) ? Math.pow((g1 + 0.055) / 1.055, 2.4) : g1 / 12.92
  b1 = (b1 > 0.04045) ? Math.pow((b1 + 0.055) / 1.055, 2.4) : b1 / 12.92

  x = (r1 * 0.4124 + g1 * 0.3576 + b1 * 0.1805) / 0.95047
  y = (r1 * 0.2126 + g1 * 0.7152 + b1 * 0.0722) / 1.00000
  z = (r1 * 0.0193 + g1 * 0.1192 + b1 * 0.9505) / 1.08883

  x = (x > 0.008856) ? Math.pow(x, 1 / 3) : (7.787 * x) + 16 / 116
  y = (y > 0.008856) ? Math.pow(y, 1 / 3) : (7.787 * y) + 16 / 116
  z = (z > 0.008856) ? Math.pow(z, 1 / 3) : (7.787 * z) + 16 / 116

  const clab = [(116 * y) - 16, 500 * (x - y), 200 * (y - z)]
  const d = Math.pow(lab[0] - clab[0], 2) + Math.pow(lab[1] - clab[1], 2) + Math.pow(lab[2] - clab[2], 2)
  return Math.sqrt(d)
}).setOutput([256])

console.log(tmp(128, 139, 117, [40.1332, 10.99816, 5.216413]))

edited Sep 25, 2020 at 1:10

answered Sep 25, 2020 at 0:52

user2258152

6855 silver badges13 bronze badges

Sign up to request clarification or add additional context in comments.

9 Comments

Blizzardengle Over a year ago

Sorry pow is leftover code I shouldn't have shown. It uses Math,pow in my actual code.

user2258152 Over a year ago

What about let x, y, z ? I think it might fix the Identifier is not defined

Blizzardengle Over a year ago

Leads to a new error: Uncaught Error: Error compiling fragment shader: ERROR: 0:466: 'assign' : l-value required (can't modify a uniform "user_r") I think I understand what this is saying...

user2258152 Over a year ago

Can you provide the values you are passing in the function r, g, b and chromaColors

Blizzardengle Over a year ago

That will depend on the frame but: 128, 139, 117, [...] The array is just the Delta values to compare against. So something like: [ 40.1332, 10.99816, 5.216413]

|

Blizzardengle · Accepted Answer · 2020-09-25 19:18:09Z

Well this is not the answer to my original question I did come up with a computationally fast poor mans alternative. I'm including this code here for anyone else stuck trying to do chroma keying in JavaScript. Visually speaking the output video is very close to the way heavier Delta E 76 code in the OP.

Step 1: Convert RGB to YUV
I found a StackOverflow answer that has a very fast RGB to YUV conversion function written in C. Later I also found Greenscreen Code and Hints by Edward Cannon that had a C function to convert RGB to YCbCr. I took both of these, converted them to JavaScript, and tested which was actually better for chroma keying. Well Edward Cannon's function was useful it did not prove any better than Camille Goudeseune's code; the SO answer reference above. Edward's code is commented out below:

let rgbToYuv = function( r, g, b ) {
    let y =  0.257 * r + 0.504 * g + 0.098 * b +  16;
    //let y =  Math.round( 0.299 * r + 0.587 * g + 0.114 * b );
    let u = -0.148 * r - 0.291 * g + 0.439 * b + 128;
    //let u = Math.round( -0.168736 * r - 0.331264 * g + 0.5 * b + 128 );
    let v =  0.439 * r - 0.368 * g - 0.071 * b + 128;
    //let v =  Math.round( 0.5 * r - 0.418688 * g - 0.081312 * b + 128 );
    return [ y, u, v ];
}

Step 2: Check How Close Two YUV Colors Are
Thanks again to Greenscreen Code and Hints by Edward Cannon comparing two YUV colors was fairly simple. We can ignore Y here and only need the U and V values; if you want to know why you will need to study up on YUV (YCbCr), particularly the section on luminance and chrominance. Here is the C code converted to JavaScript:

let colorClose = function( u, v, cu, cv ){
    return Math.sqrt( ( cu - u ) * ( cu - u ) + ( cv - v ) * ( cv - v ) );
};

If you read the article you'll notice this is not the full function. In my application I'm dealing with video not a still image so supplying a background and foreground color to include in the calculation would be difficult. It would also add to the computational load. There is a simple work around for this in the next step.

Step 3: Check Tolerance & Clean Edges
Since we're dealing with video here we loop through the pixel data for each frame and check if the colorClose value is below a certain threshold. If the color we just checked is below the tolerance level we need to turn that pixels opacity to 0 making it transparent.

Since this is a very fast poor mans chroma key we tend to get color bleed on the edges of the remaining image. Adjusting up or down on the tolerance value does a lot to reduce this but we can also add a simple feathering effect. If a pixel was not marked for transparency but is close to the tolerance level, we can partial turn it off. The code below demonstrates this:

// ...My app specific code.

/*
NOTE: chromaColors is an array holding RGB colors the user has
selected from the video feed. My app requires the user to select
the lightest and darkest part of their green screen. If lighting
is bad they can add more colors to this array and we will do our
best to chroma key them out.
*/

// Grab the current frame data from our Canvas.
let frame  = ctxHolder.getImageData( 0, 0, width, height );
let frames = frame.data.length / 4;
let colors = chromaColors.length - 1;

// Loop through every pxel of this frame.
for ( let i = 0; i < frames; i++ ) {
  
  // Each pixel is stored as an rgba value; we don't need a.
  let r = frame.data[ i * 4 + 0 ];
  let g = frame.data[ i * 4 + 1 ];
  let b = frame.data[ i * 4 + 2 ];
  
  let yuv = rgbToYuv( r, g, b );
  
  // Check the current pixel against our list of colors to turn transparent.
  for ( let c = 0; c < colors; c++ ) {
    
    // When the user selected a color for chroma keying we wen't ahead
    // and saved the YUV value to save on resources. Pull it out for use.
    let cc = chromaColors[c].yuv;
    
    // Calc the closeness (distance) of the currnet pixel and chroma color.
    let d = colorClose( yuv[1], yuv[2], cc[1], cc[2] );
    
    if( d < tolerance ){
        // Turn this pixel transparent.
        frame.data[ i * 4 + 3 ] = 0;
        break;
    } else {
      // Feather edges by lowering the opacity on pixels close to the tolerance level.
      if ( d - 1 < tolerance ){
          frame.data[ i * 4 + 3 ] = 0.1;
          break;
      }
      if ( d - 2 < tolerance ){
          frame.data[ i * 4 + 3 ] = 0.2;
          break;
      }
      if ( d - 3 < tolerance ){
          frame.data[ i * 4 + 3 ] = 0.3;
          break;
      }
      if ( d - 4 < tolerance ){
          frame.data[ i * 4 + 3 ] = 0.4;
          break;
      }
      if ( d - 5 < tolerance ){
          frame.data[ i * 4 + 3 ] = 0.5;
          break;
      }
    }
  }
}

// ...My app specific code.

// Put the altered frame data back into the video feed.
ctxMain.putImageData( frame, 0, 0 );

Additional Resources I should mention that Real-Time Chroma Key With Delta E 76 and Delta E 101 by Zachary Schuessler were a great help in getting me to these solutions.

Collectives™ on Stack Overflow

How to pass off heavy JavaScript math operations to GPU with GPU.js

2 Answers 2

9 Comments

Comments

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

9 Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Linked

Related