I am trying to draw an outline around any arbitrary object using OpenGL and shaders with a different color than the original object, while also retaining compatibility with alpha values <1. I am currently trying to get this working with just a rectangle. I tried using points to determine x, y, width, and height, but these would then fail to be transformed by view and projection matrices. Using triangles renders a regular rectangle, but I can't figure out how to create an outline around it. While I am making a 2D engine, being able to transform into 3D (ex: spinning the object on all 3 axes while retaining the outline, but not actually rendering 3D objects) is important to me.
Here is my current vertex array:
float vertices[8] = {
0.5f, 0.5f,
0.5f, -0.5f,
-0.5f, -0.5f,
-0.5f, 0.5f
};
Here is my vertex shader:
#version 330 core
layout (location = 0) in vec2 inPos;
uniform mat4 transform;
uniform mat4 view;
uniform mat4 projection;
uniform vec4 vsColor;
uniform float vsOutline;
uniform vec4 vsOutlineColor;
out VS_OUT {
vec4 color;
vec4 outlineColor;
float outlineWidth;
} vs_out;
void main() {
gl_Position = projection * view * transform * vec4(inPos, 0.0, 1.0);
vs_out.color = vsColor;
vs_out.outlineColor = vsOutlineColor;
vs_out.outlineWidth = vsOutline;
}
Here is my geometry shader, which currently just acts as a passthrough:
#version 330 core
layout (triangles) in;
layout (triangle_strip, max_vertices = 3) out;
out vec4 fsColor;
in VS_OUT {
vec4 color;
vec4 outlineColor;
float outlineWidth;
} gs_in[];
void main() {
// Draw main triangle (passthrough)
fsColor = gs_in[0].color;
gl_Position = gl_in[0].gl_Position;
EmitVertex();
gl_Position = gl_in[1].gl_Position;
EmitVertex();
gl_Position = gl_in[2].gl_Position;
EmitVertex();
// ===UNKNOWN===
// Outline the object (WIP)
fsColor = gs_in[0].outlineColor;
EndPrimitive();
}
Finally, here is my fragment shader:
#version 330 core
in vec4 fsColor;
out vec4 fragColor;
void main() {
fragColor = fsColor;
if (fragColor.a == 0) discard;
}
I am unsure if I need to do outlining in the geometry or fragment shader. I would like to have an implementation that could work with non-regular shapes (such as sprites), but that is not required at this time. However, it does have to work with other shapes such as circles and triangles.
Here is an image of the current state of my engine, with the pink rectangle needing to be outlined:

The box texture shows how objects may be transformed in the game, as well as the necessity of alpha compatibility.
UPDATE 1
Here is my render function:
void c2m::client::gl::Sprite::render() {
// Set shader uniforms if the shader is initialized
if (shader != nullptr) {
if (outlineWidth > 0) {
glStencilFunc(GL_ALWAYS, 1, 0xFF);
glStencilMask(0xFF);
}
applyTransforms();
shader->useShader();
shader->setMat4("transform", trans);
shader->setVec4("color", color.asVec4());
}
// Rebind the VAO to be able to modify its VBOs
glBindVertexArray(vao);
// Reset vertex data to class array
glBindBuffer(GL_ARRAY_BUFFER, vbo);
glBufferSubData(GL_ARRAY_BUFFER, 0, sizeof(vertices), vertices);
glBindBuffer(GL_ARRAY_BUFFER, 0);
// Bind texture
tex->bind();
// Draw
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
// If the outline width is 0 return
if (outlineWidth == 0) {
return;
}
// Draw outline
if (outlineShader != nullptr) {
glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
glStencilMask(0x00);
glDisable(GL_DEPTH_TEST);
outlineShader->useShader();
outlineShader->setVec4("color", outlineRGBA.asVec4());
// Temporary transform matrix to prevent pollution of user-set transforms
glm::mat4 tempTransform = trans;
tempTransform = glm::scale(tempTransform, glm::vec3(outlineWidth + 1, outlineWidth + 1, outlineWidth + 1));
outlineShader->setMat4("transform", tempTransform);
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
glStencilMask(0xFF);
glEnable(GL_DEPTH_TEST);
glStencilFunc(GL_ALWAYS, 1, 0xFF);
}
}
And here is the result of the function:

UPDATE 2
Here is my initialization and clear code:
Init:
// Enable various OpenGL functions
// Enable depth testing/z-indexing
glEnable(GL_DEPTH_TEST);
glDepthFunc(GL_LESS);
// Stencil
glEnable(GL_STENCIL_TEST);
// Disable stencil writing by default, to be enabled per draw cycle
glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
// Alpha
glEnable(GL_BLEND);
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
Clear code (run every draw cycle):
// clear the buffers
glClearColor(0.1f, 0.1f, 0.1f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
The only other code related to stencils is in the render() function shown above.