Need a new Latex command for square brackets

If you're able to use LuaLaTeX, you can freely modify any glyph in the font, so you can stretch the square brackets to give the results you're looking for:

\documentclass{article}

\usepackage{luacode}
\begin{luacode*}
    -----------------------------
    --- Adjustable Parameters ---
    -----------------------------

    -- We can specify the adjustments for each character in each font using
    -- the following table. All units are relative to the original width of the
    -- glyph.
    local horizontal_adjustments = {
        ["latinmodern-math.otf"] = {
            ["["] = {
                middle = 0.6,
                left_stretch = 1.0,
                right_stretch = 5.0,
                left_sidebearing = 0.0,
                right_sidebearing = -0.75,
            },
            ["]"] = {
                middle = 0.4,
                left_stretch = 5.0,
                right_stretch = 1.0,
                left_sidebearing = -0.5,
                right_sidebearing = 0.0,
            },
        },
        ["texgyrepagella-math.otf"] = {
            ["("] = {
                middle = 0.5,
                left_stretch = 1.0,
                right_stretch = 3.0,
                left_sidebearing = 0.0,
                right_sidebearing = -0.5,
            },
            [")"] = {
                middle = 0.5,
                left_stretch = 3.0,
                right_stretch = 1.0,
                left_sidebearing = -0.5,
                right_sidebearing = 0.0,
            },
        },
        ["texgyrepagella-regular.otf"] = {
            ["r"] = {
                middle = 0.6,
                left_stretch = 1.0,
                right_stretch = 5.0,
                left_sidebearing = 0.0,
                right_sidebearing = -0.7,
            },
            ["L"] = {
                middle = 0.6,
                left_stretch = 1.0,
                right_stretch = 5.0,
                left_sidebearing = 0.0,
                right_sidebearing = -0.5,
            },
        },
    }

    ----------------------
    --- Implementation ---
    ----------------------

    -- Load the font-cff module from ConTeXt, which is required to use
    -- "pack_result_tagged"
    do
        local data = io.loaddata(kpse.find_file("font-cff.lmt"))
        data = data:gsub("<const>", ""):gsub("pack_result_tagged =", "fonts.handlers.otf.pack_result_tagged =")
        load(data, "font-cff.lua", "t", luaotfload.fontloader)()
    end

    -- Cache the seen segments so that we only stretch each unique shape once.
    local seen = {}

    -- Stretch the glyph segments according to the provided adjustment
    -- parameters.
    local function stretch_glyph(segments, original_width, adjustment)
        -- Extract the parameters from the adjustment table.
        local middle = adjustment.middle * original_width
        local left_stretch = adjustment.left_stretch
        local right_stretch = adjustment.right_stretch
        local left_sidebearing = adjustment.left_sidebearing
        local right_sidebearing = adjustment.right_sidebearing

        -- Calculate the overall scale factor
        local scale_factor = (left_stretch + right_stretch) / 2
        scale_factor = scale_factor + left_sidebearing + right_sidebearing

        -- If we've already seen this shape, return the cached result.
        if seen[segments] then
            return segments, scale_factor
        end
        seen[segments] = true

        -- Loop over each segment and stretch the x-coordinates according to
        -- the provided parameters.
        for _, segment in ipairs(segments) do
            for i = 1, #segment - 1, 2 do
                -- Get the x and y coordinates of the current point.
                local x, y = segment[i], segment[i + 1]
                if type(x) ~= "number" or type(y) ~= "number" then
                    goto continue
                end

                -- Stretch the x-coordinate
                if x < middle then
                    segment[i] = left_stretch * (x - middle) + middle
                else
                    segment[i] = right_stretch * (x - middle) + middle
                end

                -- Offset the x-coordinate to keep the glyph centered
                segment[i] = segment[i] + (left_stretch - 1) * middle

                -- Adjust the left sidebearing
                segment[i] = segment[i] + (left_sidebearing * original_width)

                ::continue::
            end
        end
        return segments, scale_factor
    end

    luatexbase.add_to_callback(
        "luaotfload.patch_font",
        function(tfmdata, specification, font_id)
            -- Only apply the adjustments if the font filename matches one of
            -- the keys in the adjustments table.
            local path = tfmdata.specification.filename
            local filename = file.basename(path)
            local by_filename = horizontal_adjustments[filename]

            if not by_filename then
                return
            end

            -- Parameters that apply to the whole font.
            tfmdata.streamprovider = 1
            local size = tfmdata.size
            local units_per_em = tfmdata.units_per_em

            -- Loop over each character in the adjustments table for this font.
            for character, adjustment in pairs(by_filename) do
                -- Get the original data from the font.
                local codepoint = utf8.codepoint(character)
                local index = tfmdata.characters[codepoint].index

                local character = tfmdata.characters[codepoint]
                local shapes    = fonts.hashes.shapes[font_id].glyphs
                local streams   = fonts.hashes.streams[font_id].streams

                local original_shape = shapes[index]

                -- Stretch the glyph segments according to the provided
                -- adjustment parameters.
                local new_segments, scale_factor = stretch_glyph(
                    original_shape.segments,
                    original_shape.width,
                    adjustment
                )

                -- Update the font data with the new segments and width.
                local new_stream = fonts.handlers.otf.pack_result_tagged(
                    new_segments, scale_factor * original_shape.width, 0, 0
                )
                streams[index] = new_stream
                character.width = scale_factor * character.width
            end
        end,
        "rewrite-characters"
    )
\end{luacode*}


%%%%%%%%%%%%%%%%%%%%%
%%% Demonstration %%%
%%%%%%%%%%%%%%%%%%%%%

\usepackage{fontspec}
\usepackage{unicode-math}

\pagestyle{empty}
\setlength{\parskip}{\baselineskip}
\setlength{\parindent}{0pt}

\begin{document}
    \Huge

    \setmathfont{Latin Modern Math}
    \setmainfont{Latin Modern Roman}
    $\displaystyle [a, b]^{[a, b]^{[a, b]}}$

    \setmathfont{TeX Gyre Pagella Math}
    \setmainfont{TeX Gyre Pagella}
    $\displaystyle (a, b)_{(a, b)_{(a, b)}}$

    really Long
\end{document}