Faster approach to iterative pixel mapping in Unity

Question

I need to take an initial Texture2D and, depending on four sliders, remap the RGBA channels. What I currently have works, but is not very performant, especially with larger images.

for (int x = 0; x < _tempIn.width; x++)
   for (int y = 0; y < _tempIn.height; y++)
   {
      _tempColorHold = _tempIn.GetPixel(x, y);

      r = (rSlider.value == (int)RGBA051.zero) ? 0f :
          (rSlider.value == (int)RGBA051.r) ? _tempColorHold.r :
          (rSlider.value == (int)RGBA051.g) ? _tempColorHold.g :
          (rSlider.value == (int)RGBA051.half) ? 0.5f :
          (rSlider.value == (int)RGBA051.b) ? _tempColorHold.b :
          (rSlider.value == (int)RGBA051.a) ? _tempColorHold.a :
          1f;

      g = (gSlider.value == (int)RGBA051.zero) ? 0f :
          (gSlider.value == (int)RGBA051.r) ? _tempColorHold.r :
          (gSlider.value == (int)RGBA051.g) ? _tempColorHold.g :
          (gSlider.value == (int)RGBA051.half) ? 0.5f :
          (gSlider.value == (int)RGBA051.b) ? _tempColorHold.b :
          (gSlider.value == (int)RGBA051.a) ? _tempColorHold.a :
          1f;

      b = (bSlider.value == (int)RGBA051.zero) ? 0f :
          (bSlider.value == (int)RGBA051.r) ? _tempColorHold.r :
          (bSlider.value == (int)RGBA051.g) ? _tempColorHold.g :
          (bSlider.value == (int)RGBA051.half) ? 0.5f :
          (bSlider.value == (int)RGBA051.b) ? _tempColorHold.b :
          (bSlider.value == (int)RGBA051.a) ? _tempColorHold.a :
          1f;

      a = (aSlider.value == (int)RGBA051.zero) ? 0f :
          (aSlider.value == (int)RGBA051.r) ? _tempColorHold.r :
          (aSlider.value == (int)RGBA051.g) ? _tempColorHold.g :
          (aSlider.value == (int)RGBA051.half) ? 0.5f :
          (aSlider.value == (int)RGBA051.b) ? _tempColorHold.b :
          (aSlider.value == (int)RGBA051.a) ? _tempColorHold.a :
          1f;

      _tempOut.SetPixel(x, y, new Color(r, g, b, a));
   }

Basically RGBA051 is an enum like this:

public enum RGBA051 { zero, r, g, half, b, a, one }

There are four int sliders that have have a value 0-7 selected, mapping to that enum. If the red channel has zero set, the red channel for all pixels will be set to zero. If the red channel has r set, it will get the red channel value for each pixel (it will remain the same), and so forth.

During the execution of this method, the slider cannot change, so the value of rSlider and gSlider and such will remain the same throughout the loop. It seems redundant to execute ternary operator during the assignment of each pixel, but I can't quite wrap my head around how to unpack that check while still setting each pixel properly.

This looks like something that would be far more performant to do with a shader. — Philipp
– Philipp, Commented Feb 8, 2023 at 20:38

DMGregory · Accepted Answer · 2023-02-09 18:05:38Z

The operations you want to perform can be expressed as a single matrix multiplication plus an offset. You'll also get better speed operating on all pixels in one fell swoop, rather than getting and setting them one at a time.

Vector4 ToMatrixRow(RGBA051 value) {
    switch (value) {
        case RGBA051.r: return new Vector4(1, 0, 0, 0);
        case RGBA051.g: return new Vector4(0, 1, 0, 0);
        case RGBA051.b: return new Vector4(0, 0, 1, 0);
        case RGBA051.a: return new Vector4(0, 0, 0, 1);
        default: return Vector4.zero;
    }
}

float ToOffset(RGBA051 value) {
    switch(value) {
        case RGBA051.half: return 0.5f;
        case RGBA051.one : return 1.0f;
        default: return 0.0f;
    }
}

void RecolourTexture(Texture2D source, Texture2D destination,
                     RGBA051 r, RGBA051 g, RGBA051 b, RGBA051 a) {

    // Get all pixel data at once.
    // (To avoid allocation, you can use GetPixelData<T>(), but 
    //  that requires different handling for each texture format)
    var pixelData = source.GetPixels32();

    // Construct a matrix with rows that route the desired input
    // channel to the desired output channel, or zero the output.
    var matrix = new Matrix4x4(
        ToMatrixRow(r),
        ToMatrixRow(g),
        ToMatrixRow(b),
        ToMatrixRow(a)
    );
    matrix = matrix.transpose;

    // Compute a vector of constants for the half/one values.
    var offset = new Vector4(ToOffset(r), ToOffset(g), ToOffset(b), ToOffset(a));


    for(int i =0; i < pixelData.Length; i++) {
        // Unpack 4 bytes 0-255 into 4 floats 0-1.
        Vector4 c = (Color)pixelData[i];

        // Apply the colour transformation.
        c = (matrix * c) + offset;

        // Convert the result back to the compact format.
        pixelData[i] = (Color)c;
    }

    // Set all pixels at once.
    destination.SetPixels32(pixelData);
    // Apply modified pixels 
    // (false values here skip updating mipmaps & keep texture CPU-readable)
    destination.Apply(false, false);
}

As Philipp points out, this would be even faster to do in a shader, operating on all pixels in the image in parallel. You can do that with Graphics.Blit(), using the same matrix multiply and add technique in the fragment shader code.

Another thought that just occurred to me: since you only ever want to swap or wholly replace a channel, you could operate on them like this:

var channelValues = new byte[7];
channelValues[(int)RGBA051.zero] = 0;
channelValues[(int)RGBA051.one] = 255;
channelValues[(int)RGBA051.half] = 128;

// ... then, inside the pixel loop:

Color32 c = pixelData[i];

channelValues[(int)RGBA051.r] = c.r;
channelValues[(int)RGBA051.g] = c.g;
channelValues[(int)RGBA051.b] = c.b;
channelValues[(int)RGBA051.a] = c.a;

c.r = channelValues[(int)r];
c.g = channelValues[(int)g];
c.b = channelValues[(int)b];
c.a = channelValues[(int)a];

pixelData[i] = c;

That's 8 byte assignments instead of 4 byte-to-float conversions, 16 float multiplies, 16 float adds, and 4 float-to-byte conversions, so I'd expect it should run even faster, barring some pipelining shenanigans.

The advantage of the full matrix version above is that it's even more general, and can handle other image adjustments like brightness/contrast, desaturation, even a version of hue shifting, just by initializing the matrix with different coefficients.

Jesse Williams · Accepted Answer · 2023-02-11 17:51:34Z

For additional testing, I've created a handful of CPU and ComputeShader methods as shown below. I definitely welcome any improvements in the comments.

Texture2D_Mod.compute:

#pragma kernel ChannelSwapFunc
#pragma kernel ChannelSwap4x4

RWTexture2D<float4> result;
Texture2D<float4> input;
uint channelSwapR;
uint channelSwapG;
uint channelSwapB;
uint channelSwapA;
matrix swapmatrix;
float4 swapmatrixoffset;


[numthreads(8,8,1)]
void ChannelSwapFunc(uint3 id : SV_DispatchThreadID)
{
    float r;
    float g;
    float b;
    float a;

    if (channelSwapR == 4)
        r = 0;
    else if (channelSwapR == 5)
        r = 0.5;
    else if (channelSwapR == 6)
        r = 1;
    else
        r = input[id.xy][channelSwapR];

    if (channelSwapG == 4)
        g = 0;
    else if (channelSwapG == 5)
        g = 0.5;
    else if (channelSwapG == 6)
        g = 1;
    else
        g = input[id.xy][channelSwapG];

    if (channelSwapB == 4)
        b = 0;
    else if (channelSwapB == 5)
        b = 0.5;
    else if (channelSwapB == 6)
        b = 1;
    else
        b = input[id.xy][channelSwapB];

    if (channelSwapA == 4)
        a = 0;
    else if (channelSwapA == 5)
        a = 0.5;
    else if (channelSwapA == 6)
        a = 1;
    else
        a = input[id.xy][channelSwapA];

    result[id.xy] = float4(r, g, b, a);
}

[numthreads(8, 8, 1)]
void ChannelSwap4x4(uint3 id : SV_DispatchThreadID)
{
    result[id.xy] = mul(input[id.xy], swapmatrix) + swapmatrixoffset;
}

PerformFunction() - this is the standard CPU method, my initial attempt. Average time was 854.6ms:

private void PerformFunction()
    {
        if (valueIn != null)
        {
            float r;
            float g;
            float b;
            float a;

            Texture2D _tempIn = valueIn.DuplicateTexture();

            Color _tempColorHold = new Color();

            Texture2D _tempOut = new Texture2D(_tempIn.width, _tempIn.height);
            _tempOut.alphaIsTransparency = true;

            for (int x = 0; x < _tempIn.width; x++)
                for (int y = 0; y < _tempIn.height; y++)
                {
                    _tempColorHold = _tempIn.GetPixel(x, y);

                    r = (rSlider.value == (int)RGBA.r)    ? _tempColorHold.r :
                        (rSlider.value == (int)RGBA.g)    ? _tempColorHold.g :
                        (rSlider.value == (int)RGBA.b)    ? _tempColorHold.b :
                        (rSlider.value == (int)RGBA.a)    ? _tempColorHold.a :
                        (rSlider.value == (int)RGBA.zero) ? 0f               :
                        (rSlider.value == (int)RGBA.half) ? 0.5f             : 1f;

                    g = (gSlider.value == (int)RGBA.r)    ? _tempColorHold.r :
                        (gSlider.value == (int)RGBA.g)    ? _tempColorHold.g :
                        (gSlider.value == (int)RGBA.b)    ? _tempColorHold.b :
                        (gSlider.value == (int)RGBA.a)    ? _tempColorHold.a :
                        (gSlider.value == (int)RGBA.zero) ? 0f               :
                        (gSlider.value == (int)RGBA.half) ? 0.5f             : 1f;

                    b = (bSlider.value == (int)RGBA.r)    ? _tempColorHold.r :
                        (bSlider.value == (int)RGBA.g)    ? _tempColorHold.g :
                        (bSlider.value == (int)RGBA.b)    ? _tempColorHold.b :
                        (bSlider.value == (int)RGBA.a)    ? _tempColorHold.a :
                        (bSlider.value == (int)RGBA.zero) ? 0f               :
                        (bSlider.value == (int)RGBA.half) ? 0.5f             : 1f;

                    a = (aSlider.value == (int)RGBA.r)    ? _tempColorHold.r :
                        (aSlider.value == (int)RGBA.g)    ? _tempColorHold.g :
                        (aSlider.value == (int)RGBA.b)    ? _tempColorHold.b :
                        (aSlider.value == (int)RGBA.a)    ? _tempColorHold.a :
                        (aSlider.value == (int)RGBA.zero) ? 0f               :
                        (aSlider.value == (int)RGBA.half) ? 0.5f             : 1f;

                    _tempOut.SetPixel(x, y, new Color(r, g, b, a));
                }

            _tempOut.Apply(false, false);

            this.ValueOut = _tempOut;
            portOut.Value = this.ValueOut;
        }
    }

PerformFunctionCompute() - this was the same basic principle, but run using ChannelSwapFunc() in the ComputeShader. Average time was 55.2ms:

private void PerformFunctionCompute()
    {
        shader = (ComputeShader)Resources.Load("Texture2D_Mod");
        int _kernelHandle = shader.FindKernel("ChannelSwapFunc");
        Texture2D _tempIn = valueIn.DuplicateTexture();
        RenderTexture _tex = new RenderTexture(_tempIn.width, _tempIn.height, 24);
        _tex.enableRandomWrite = true;
        _tex.Create();

        shader.SetTexture(_kernelHandle, "result", _tex);
        shader.SetTexture(_kernelHandle, "input", _tempIn);
        shader.SetInt("channelSwapR", rSlider.value);
        shader.SetInt("channelSwapG", gSlider.value);
        shader.SetInt("channelSwapB", bSlider.value);
        shader.SetInt("channelSwapA", aSlider.value);
        shader.Dispatch(_kernelHandle, _tempIn.width / 8, _tempIn.height / 8, 1);

        RenderTexture.active = _tex;
        Texture2D _result = new Texture2D(_tempIn.width, _tempIn.height);
        _result.ReadPixels(new Rect(0, 0, _tempIn.width, _tempIn.height), 0, 0);
        _result.Apply(false, false);

        this.ValueOut = _result;
        portOut.Value = this.ValueOut;
    }

PerformFunctionMatrix4x4() - this was the Matrix4x4 version from @DMGregory. Average time was 395.4ms:

private void PerformFunctionMatrix4x4()
    {
        if (valueIn != null)
        {
            Texture2D _tempIn = valueIn.DuplicateTexture();
            Color32[] pixelData = _tempIn.GetPixels32();

            var matrix = new Matrix4x4(
                ToMatrixRow((RGBA)rRGBA),
                ToMatrixRow((RGBA)gRGBA),
                ToMatrixRow((RGBA)bRGBA),
                ToMatrixRow((RGBA)aRGBA)
                );

            matrix = matrix.transpose;

            var offset = new Vector4(ToOffset((RGBA)rRGBA), ToOffset((RGBA)gRGBA), ToOffset((RGBA)bRGBA), ToOffset((RGBA)aRGBA));

            for (int i = 0; i < pixelData.Length; i++)
            {
                Vector4 c = (Color)pixelData[i];

                c = (matrix * c) + offset;

                pixelData[i] = (Color)c;
            }

            Texture2D _tempOut = new Texture2D(_tempIn.width, _tempIn.height);
            _tempOut.alphaIsTransparency = true;

            _tempOut.SetPixels32(pixelData);

            _tempOut.Apply(false, false);

            this.ValueOut = _tempOut;
            portOut.Value = this.ValueOut;
        }
    }

PerformFunctionComputeMatrix4x4() - this is the ComputeShader version of the Matrix4x4 version of the function. Average time was 54.5ms:

private void PerformFunctionComputeMatrix4x4()
        {
            if (valueIn != null)
            {
                shader = (ComputeShader)Resources.Load("Texture2D_Mod");
                int _kernelHandle = shader.FindKernel("ChannelSwap4x4");
                Texture2D _tempIn = valueIn.DuplicateTexture();
                RenderTexture _tex = new RenderTexture(_tempIn.width, _tempIn.height, 24);
                _tex.enableRandomWrite = true;
                _tex.Create();

                var matrix = new Matrix4x4(
                    ToMatrixRow((RGBA)rRGBA),
                    ToMatrixRow((RGBA)gRGBA),
                    ToMatrixRow((RGBA)bRGBA),
                    ToMatrixRow((RGBA)aRGBA)
                    );

                matrix = matrix.transpose;

                var offset = new Vector4(ToOffset((RGBA)rRGBA), ToOffset((RGBA)gRGBA), ToOffset((RGBA)bRGBA), ToOffset((RGBA)aRGBA));

                shader.SetTexture(_kernelHandle, "result", _tex);
                shader.SetTexture(_kernelHandle, "input", _tempIn);
                shader.SetMatrix("swapmatrix", matrix);
                shader.SetVector("swapmatrixoffset", offset);
                shader.Dispatch(_kernelHandle, _tempIn.width / 8, _tempIn.height / 8, 1);

                RenderTexture.active = _tex;
                Texture2D _result = new Texture2D(_tempIn.width, _tempIn.height);
                _result.ReadPixels(new Rect(0, 0, _tempIn.width, _tempIn.height), 0, 0);
                _result.Apply(false, false);

                this.ValueOut = _result;
                portOut.Value = this.ValueOut;
            }
        }

Stack Exchange Network

Faster approach to iterative pixel mapping in Unity

2 Answers 2

You must log in to answer this question.

Hot Network Questions

Faster approach to iterative pixel mapping in Unity

2 Answers 2

You must log in to answer this question.

Related

Hot Network Questions