r3d/shaders/post/fxaa.frag

/* fxaa.frag -- Fragment shader for applying FXAA to the scene
 *
 * Copyright (c) 2025-2026 Le Juez Victor
 *
 * This software is provided 'as-is', without any express or implied warranty.
 * For conditions of distribution and use, see accompanying LICENSE file.
 */

/*
 * Portions of this code are derived from NVIDIA's FXAA technology.
 *
 * Copyright (c) 2011 NVIDIA Corporation. All rights reserved.
 *
 * TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, THIS SOFTWARE IS PROVIDED
 * "AS IS" AND NVIDIA AND ITS SUPPLIERS DISCLAIM ALL WARRANTIES, EITHER EXPRESS
 * OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, NONINFRINGEMENT, IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT SHALL
 * NVIDIA OR ITS SUPPLIERS BE LIABLE FOR ANY DIRECT, SPECIAL, INCIDENTAL, INDIRECT,
 * OR CONSEQUENTIAL DAMAGES WHATSOEVER (INCLUDING, WITHOUT LIMITATION, DAMAGES FOR
 * LOSS OF BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF BUSINESS INFORMATION,
 * OR ANY OTHER PECUNIARY LOSS) ARISING OUT OF THE USE OF OR INABILITY TO USE THIS SOFTWARE,
 * EVEN IF NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
 */

// TODO: Add quality preset option (need to recompile shader)

#version 330 core

/* === Varyings === */

noperspective in vec2 vTexCoord;

/* === Uniforms === */

uniform sampler2D uSourceTex;
uniform vec2 uSourceTexel;

/* === Fragments === */

out vec4 FragColor;

/* === FXAA presets === */

/*
    FXAA_PRESET - Choose compile-in knob preset 0-5.
    ------------------------------------------------------------------------------
    FXAA_EDGE_THRESHOLD - The minimum amount of local contrast required 
                          to apply algorithm.
                          1.0/3.0  - too little
                          1.0/4.0  - good start
                          1.0/8.0  - applies to more edges
                          1.0/16.0 - overkill
    ------------------------------------------------------------------------------
    FXAA_EDGE_THRESHOLD_MIN - Trims the algorithm from processing darks.
                              Perf optimization.
                              1.0/32.0 - visible limit (smaller isn't visible)
                              1.0/16.0 - good compromise
                              1.0/12.0 - upper limit (seeing artifacts)
    ------------------------------------------------------------------------------
    FXAA_SEARCH_STEPS - Maximum number of search steps for end of span.
    ------------------------------------------------------------------------------
    FXAA_SEARCH_THRESHOLD - Controls when to stop searching.
                            1.0/4.0 - seems to be the best quality wise
    ------------------------------------------------------------------------------
    FXAA_SUBPIX_TRIM - Controls sub-pixel aliasing removal.
                       1.0/2.0 - low removal
                       1.0/3.0 - medium removal
                       1.0/4.0 - default removal
                       1.0/8.0 - high removal
                       0.0 - complete removal
    ------------------------------------------------------------------------------
    FXAA_SUBPIX_CAP - Insures fine detail is not completely removed.
                      This is important for the transition of sub-pixel detail,
                      like fences and wires.
                      3.0/4.0 - default (medium amount of filtering)
                      7.0/8.0 - high amount of filtering
                      1.0 - no capping of sub-pixel aliasing removal
*/

#define FXAA_PRESET 5

#if (FXAA_PRESET == 3)
    #define FXAA_EDGE_THRESHOLD      (1.0/8.0)
    #define FXAA_EDGE_THRESHOLD_MIN  (1.0/16.0)
    #define FXAA_SEARCH_STEPS        16
    #define FXAA_SEARCH_THRESHOLD    (1.0/4.0)
    #define FXAA_SUBPIX_CAP          (3.0/4.0)
    #define FXAA_SUBPIX_TRIM         (1.0/4.0)
#endif
#if (FXAA_PRESET == 4)
    #define FXAA_EDGE_THRESHOLD      (1.0/8.0)
    #define FXAA_EDGE_THRESHOLD_MIN  (1.0/24.0)
    #define FXAA_SEARCH_STEPS        24
    #define FXAA_SEARCH_THRESHOLD    (1.0/4.0)
    #define FXAA_SUBPIX_CAP          (3.0/4.0)
    #define FXAA_SUBPIX_TRIM         (1.0/4.0)
#endif
#if (FXAA_PRESET == 5)
    #define FXAA_EDGE_THRESHOLD      (1.0/8.0)
    #define FXAA_EDGE_THRESHOLD_MIN  (1.0/24.0)
    #define FXAA_SEARCH_STEPS        32
    #define FXAA_SEARCH_THRESHOLD    (1.0/4.0)
    #define FXAA_SUBPIX_CAP          (3.0/4.0)
    #define FXAA_SUBPIX_TRIM         (1.0/4.0)
#endif

#define FXAA_SUBPIX_TRIM_SCALE (1.0/(1.0 - FXAA_SUBPIX_TRIM))

/* === Helper Functions === */


float FxaaLuma(vec3 rgb)
{
    // Return the luma, the estimation of luminance from rgb inputs.
    // This approximates luma using one FMA instruction,
    // skipping normalization and tossing out blue.
    // FxaaLuma() will range 0.0 to 2.963210702.
    return rgb.y * (0.587/0.299) + rgb.x;
}

vec3 FxaaLerp3(vec3 a, vec3 b, float amountOfA)
{
    return (vec3(-amountOfA) * b) + ((a * vec3(amountOfA)) + b);
}

vec4 FxaaTexOff(sampler2D tex, vec2 pos, ivec2 off, vec2 rcpFrame)
{
    float x = pos.x + float(off.x) * rcpFrame.x;
    float y = pos.y + float(off.y) * rcpFrame.y;
    return texture(tex, vec2(x, y));
}

/* === Main function === */

void main()
{
    vec2 pos = vTexCoord;

    vec3 rgbN = FxaaTexOff(uSourceTex, pos.xy, ivec2( 0,-1), uSourceTexel).xyz;
    vec3 rgbW = FxaaTexOff(uSourceTex, pos.xy, ivec2(-1, 0), uSourceTexel).xyz;
    vec3 rgbM = FxaaTexOff(uSourceTex, pos.xy, ivec2( 0, 0), uSourceTexel).xyz;
    vec3 rgbE = FxaaTexOff(uSourceTex, pos.xy, ivec2( 1, 0), uSourceTexel).xyz;
    vec3 rgbS = FxaaTexOff(uSourceTex, pos.xy, ivec2( 0, 1), uSourceTexel).xyz;
    
    float lumaN = FxaaLuma(rgbN);
    float lumaW = FxaaLuma(rgbW);
    float lumaM = FxaaLuma(rgbM);
    float lumaE = FxaaLuma(rgbE);
    float lumaS = FxaaLuma(rgbS);
    float rangeMin = min(lumaM, min(min(lumaN, lumaW), min(lumaS, lumaE)));
    float rangeMax = max(lumaM, max(max(lumaN, lumaW), max(lumaS, lumaE)));
    
    float range = rangeMax - rangeMin;
    if(range < max(FXAA_EDGE_THRESHOLD_MIN, rangeMax * FXAA_EDGE_THRESHOLD)) {
        FragColor = vec4(rgbM, 1.0);
        return;
    }
    
    vec3 rgbL = rgbN + rgbW + rgbM + rgbE + rgbS;
    
    float lumaL = (lumaN + lumaW + lumaE + lumaS) * 0.25;
    float rangeL = abs(lumaL - lumaM);
    float blendL = max(0.0, (rangeL / range) - FXAA_SUBPIX_TRIM) * FXAA_SUBPIX_TRIM_SCALE; 
    blendL = min(FXAA_SUBPIX_CAP, blendL);
    
    vec3 rgbNW = FxaaTexOff(uSourceTex, pos.xy, ivec2(-1,-1), uSourceTexel).xyz;
    vec3 rgbNE = FxaaTexOff(uSourceTex, pos.xy, ivec2( 1,-1), uSourceTexel).xyz;
    vec3 rgbSW = FxaaTexOff(uSourceTex, pos.xy, ivec2(-1, 1), uSourceTexel).xyz;
    vec3 rgbSE = FxaaTexOff(uSourceTex, pos.xy, ivec2( 1, 1), uSourceTexel).xyz;
    rgbL += (rgbNW + rgbNE + rgbSW + rgbSE);
    rgbL *= vec3(1.0/9.0);
    
    float lumaNW = FxaaLuma(rgbNW);
    float lumaNE = FxaaLuma(rgbNE);
    float lumaSW = FxaaLuma(rgbSW);
    float lumaSE = FxaaLuma(rgbSE);
    
    float edgeVert = 
        abs((0.25 * lumaNW) + (-0.5 * lumaN) + (0.25 * lumaNE)) +
        abs((0.50 * lumaW ) + (-1.0 * lumaM) + (0.50 * lumaE )) +
        abs((0.25 * lumaSW) + (-0.5 * lumaS) + (0.25 * lumaSE));
    float edgeHorz = 
        abs((0.25 * lumaNW) + (-0.5 * lumaW) + (0.25 * lumaSW)) +
        abs((0.50 * lumaN ) + (-1.0 * lumaM) + (0.50 * lumaS )) +
        abs((0.25 * lumaNE) + (-0.5 * lumaE) + (0.25 * lumaSE));
        
    bool horzSpan = edgeHorz >= edgeVert;
    float lengthSign = horzSpan ? -uSourceTexel.y : -uSourceTexel.x;
    
    if(!horzSpan) {
        lumaN = lumaW;
        lumaS = lumaE;
    }
    
    float gradientN = abs(lumaN - lumaM);
    float gradientS = abs(lumaS - lumaM);
    lumaN = (lumaN + lumaM) * 0.5;
    lumaS = (lumaS + lumaM) * 0.5;
    
    if (gradientN < gradientS) {
        lumaN = lumaS;
        lumaN = lumaS;
        gradientN = gradientS;
        lengthSign *= -1.0;
    }
    
    vec2 posN;
    posN.x = pos.x + (horzSpan ? 0.0 : lengthSign * 0.5);
    posN.y = pos.y + (horzSpan ? lengthSign * 0.5 : 0.0);
    
    gradientN *= FXAA_SEARCH_THRESHOLD;
    
    vec2 posP = posN;
    vec2 offNP = horzSpan ? vec2(uSourceTexel.x, 0.0) : vec2(0.0, uSourceTexel.y); 
    float lumaEndN = lumaN;
    float lumaEndP = lumaN;
    bool doneN = false;
    bool doneP = false;
    posN += offNP * vec2(-1.0, -1.0);
    posP += offNP * vec2( 1.0,  1.0);
    
    for(int i = 0; i < FXAA_SEARCH_STEPS; i++) {
        if(!doneN) {
            lumaEndN = FxaaLuma(texture(uSourceTex, posN.xy).xyz);
        }
        if(!doneP) {
            lumaEndP = FxaaLuma(texture(uSourceTex, posP.xy).xyz);
        }
        
        doneN = doneN || (abs(lumaEndN - lumaN) >= gradientN);
        doneP = doneP || (abs(lumaEndP - lumaN) >= gradientN);
        
        if(doneN && doneP) {
            break;
        }
        if(!doneN) {
            posN -= offNP;
        }
        if(!doneP) {
            posP += offNP;
        }
    }
    
    float dstN = horzSpan ? pos.x - posN.x : pos.y - posN.y;
    float dstP = horzSpan ? posP.x - pos.x : posP.y - pos.y;
    bool directionN = dstN < dstP;
    lumaEndN = directionN ? lumaEndN : lumaEndP;
    
    if(((lumaM - lumaN) < 0.0) == ((lumaEndN - lumaN) < 0.0)) {
        lengthSign = 0.0;
    }
 
    float spanLength = (dstP + dstN);
    dstN = directionN ? dstN : dstP;
    float subPixelOffset = (0.5 + (dstN * (-1.0/spanLength))) * lengthSign;
    vec3 rgbF = texture(uSourceTex, vec2(
        pos.x + (horzSpan ? 0.0 : subPixelOffset),
        pos.y + (horzSpan ? subPixelOffset : 0.0))).xyz;

    FragColor = vec4(FxaaLerp3(rgbL, rgbF, blendL), 1.0);
}