If you're able to use LuaLaTeX, you can freely modify any glyph in the font, so you can stretch the square brackets to give the results you're looking for:

\documentclass{article}

\usepackage{luacode}
\begin{luacode*}
    -----------------------------
    --- Adjustable Parameters ---
    -----------------------------

    -- We can specify the adjustments for each character in each font using
    -- the following table. All units are relative to the original width of the
    -- glyph.
    local horizontal_adjustments = {
        ["latinmodern-math.otf"] = {
            ["["] = {
                middle = 0.6,
                left_stretch = 1.0,
                right_stretch = 5.0,
                left_sidebearing = 0.0,
                right_sidebearing = -0.75,
            },
            ["]"] = {
                middle = 0.4,
                left_stretch = 5.0,
                right_stretch = 1.0,
                left_sidebearing = -0.5,
                right_sidebearing = 0.0,
            },
        },
        ["texgyrepagella-math.otf"] = {
            ["("] = {
                middle = 0.5,
                left_stretch = 1.0,
                right_stretch = 3.0,
                left_sidebearing = 0.0,
                right_sidebearing = -0.5,
            },
            [")"] = {
                middle = 0.5,
                left_stretch = 3.0,
                right_stretch = 1.0,
                left_sidebearing = -0.5,
                right_sidebearing = 0.0,
            },
        },
        ["texgyrepagella-regular.otf"] = {
            ["r"] = {
                middle = 0.6,
                left_stretch = 1.0,
                right_stretch = 5.0,
                left_sidebearing = 0.0,
                right_sidebearing = -0.7,
            },
            ["L"] = {
                middle = 0.6,
                left_stretch = 1.0,
                right_stretch = 5.0,
                left_sidebearing = 0.0,
                right_sidebearing = -0.5,
            },
        },
    }

    ----------------------
    --- Implementation ---
    ----------------------

    -- Load the font-cff module from ConTeXt, which is required to use
    -- "pack_result_tagged"
    do
        local data = io.loaddata(kpse.find_file("font-cff.lmt"))
        data = data:gsub("<const>", ""):gsub("pack_result_tagged =", "fonts.handlers.otf.pack_result_tagged =")
        load(data, "font-cff.lua", "t", luaotfload.fontloader)()
    end

    -- Cache the seen segments so that we only stretch each unique shape once.
    local seen = {}

    -- Stretch the glyph segments according to the provided adjustment
    -- parameters.
    local function stretch_glyph(segments, original_width, adjustment)
        -- Extract the parameters from the adjustment table.
        local middle = adjustment.middle * original_width
        local left_stretch = adjustment.left_stretch
        local right_stretch = adjustment.right_stretch
        local left_sidebearing = adjustment.left_sidebearing
        local right_sidebearing = adjustment.right_sidebearing

        -- Calculate the overall scale factor
        local scale_factor = (left_stretch + right_stretch) / 2
        scale_factor = scale_factor + left_sidebearing + right_sidebearing

        -- If we've already seen this shape, return the cached result.
        if seen[segments] then
            return segments, scale_factor
        end
        seen[segments] = true

        -- Loop over each segment and stretch the x-coordinates according to
        -- the provided parameters.
        for _, segment in ipairs(segments) do
            for i = 1, #segment - 1, 2 do
                -- Get the x and y coordinates of the current point.
                local x, y = segment[i], segment[i + 1]
                if type(x) ~= "number" or type(y) ~= "number" then
                    goto continue
                end

                -- Stretch the x-coordinate
                if x < middle then
                    segment[i] = left_stretch * (x - middle) + middle
                else
                    segment[i] = right_stretch * (x - middle) + middle
                end

                -- Offset the x-coordinate to keep the glyph centered
                segment[i] = segment[i] + (left_stretch - 1) * middle

                -- Adjust the left sidebearing
                segment[i] = segment[i] + (left_sidebearing * original_width)

                ::continue::
            end
        end
        return segments, scale_factor
    end

    luatexbase.add_to_callback(
        "luaotfload.patch_font",
        function(tfmdata, specification, font_id)
            -- Only apply the adjustments if the font filename matches one of
            -- the keys in the adjustments table.
            local path = tfmdata.specification.filename
            local filename = file.basename(path)
            local by_filename = horizontal_adjustments[filename]

            if not by_filename then
                return
            end

            -- Parameters that apply to the whole font.
            tfmdata.streamprovider = 1
            local size = tfmdata.size
            local units_per_em = tfmdata.units_per_em

            -- Loop over each character in the adjustments table for this font.
            for character, adjustment in pairs(by_filename) do
                -- Get the original data from the font.
                local codepoint = utf8.codepoint(character)
                local index = tfmdata.characters[codepoint].index

                local character = tfmdata.characters[codepoint]
                local shapes    = fonts.hashes.shapes[font_id].glyphs
                local streams   = fonts.hashes.streams[font_id].streams

                local original_shape = shapes[index]

                -- Stretch the glyph segments according to the provided
                -- adjustment parameters.
                local new_segments, scale_factor = stretch_glyph(
                    original_shape.segments,
                    original_shape.width,
                    adjustment
                )

                -- Update the font data with the new segments and width.
                local new_stream = fonts.handlers.otf.pack_result_tagged(
                    new_segments, scale_factor * original_shape.width, 0, 0
                )
                streams[index] = new_stream
                character.width = scale_factor * character.width
            end
        end,
        "rewrite-characters"
    )
\end{luacode*}


%%%%%%%%%%%%%%%%%%%%%
%%% Demonstration %%%
%%%%%%%%%%%%%%%%%%%%%

\usepackage{fontspec}
\usepackage{unicode-math}

\pagestyle{empty}
\setlength{\parskip}{\baselineskip}
\setlength{\parindent}{0pt}

\begin{document}
    \Huge

    \setmathfont{Latin Modern Math}
    \setmainfont{Latin Modern Roman}
    $\displaystyle [a, b]^{[a, b]^{[a, b]}}$

    \setmathfont{TeX Gyre Pagella Math}
    \setmainfont{TeX Gyre Pagella}
    $\displaystyle (a, b)_{(a, b)_{(a, b)}}$

    really Long
\end{document}