How can I calculate vertex positions for a rectangle outline using a unit square, vertex normals, model matrix and outline thickness?

I have updated my Unity project which I think does what you want: https://github.com/paulsinnett/RectangleOutlineShader

As long as you stick to integer coordinates, scales, and widths, I think it should be solid.

Here is the new code to generate the geometry:

using UnityEngine;

public class OutlineBox : MonoBehaviour
{
    void Start()
    {
        MeshFilter meshFilter = GetComponent<MeshFilter>();
        Mesh mesh = new Mesh
        {
            name = "OutlineBox"
        };
        meshFilter.mesh = mesh;

        mesh.vertices = new Vector3[]
        {
            new Vector3(0.0f, 1.0f, 0f),
            new Vector3(1.0f, 1.0f, 0f),
            new Vector3(1.0f, 0.0f, 0f),
            new Vector3(0.0f, 0.0f, 0f),
            new Vector3(0.0f, 1.0f, 0f),
            new Vector3(1.0f, 1.0f, 0f),
            new Vector3(1.0f, 0.0f, 0f),
            new Vector3(0.0f, 0.0f, 0f)
        };

        mesh.uv = new Vector2[]
        {
            // note: in clip space y is upside down
            new Vector2(-1.0f, -1.0f),
            new Vector2(1.0f, -1.0f),
            new Vector2(1.0f, 1.0f),
            new Vector2(-1.0f, 1.0f),
            new Vector2(0.0f, 0.0f),
            new Vector2(0.0f, 0.0f),
            new Vector2(0.0f, 0.0f),
            new Vector2(0.0f, 0.0f)
        };

        mesh.triangles = new int[]
        {
            0, 1, 5,
            0, 5, 4,
            1, 2, 6,
            1, 6, 5,
            2, 3, 7,
            2, 7, 6,
            3, 0, 4,
            3, 4, 7
        };
    }
}

Notes: I've switched to passing the mitre data into the uv channel because I think the normal data might get corrupted by vertex compression if its not length 1 data. Also the y directions are reversed as in Unity, the clip space is the other way up.

Here is the modified shader code:

Shader "Unlit/OutlineBoxShader"
{
    Properties
    {
        _Width ("Outline Width", Range(0, 10)) = 1
    }
    SubShader
    {
        Tags { "RenderType"="Opaque" }
        LOD 100

        Pass
        {
            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag

            #include "UnityCG.cginc"

            struct appdata
            {
                float4 vertex : POSITION;
                float2 uv : TEXCOORD0;
            };
            
            struct v2f
            {
                float4 vertex : SV_POSITION;
            };

            float _Width;

            v2f vert (appdata v)
            {
                v2f o;
                float4 clip = UnityObjectToClipPos(v.vertex);
                // in Unity clip space is -1 to 1 so pixels are twice the size
                float2 pixelSize = 2 * (_ScreenParams.zw - float2(1, 1));
                clip.xy += v.uv * _Width * pixelSize;
                o.vertex = clip;
                return o;
            }

            fixed4 frag (v2f i) : SV_Target
            {
                return fixed4(1, 1, 1, 1);
            }
            ENDCG
        }
    }
}

Notes: the \$\times 2\$ is needed to calculate the pixel size in Unity because in clip space the screen is -1 to 1, whereas the screen space it is 0 to 1.