If you're able to use LuaLaTeX, you can freely modify any glyph in the font, so you can stretch the square brackets to give the results you're looking for:
\documentclass{article}
\usepackage{luacode}
\begin{luacode*}
-----------------------------
--- Adjustable Parameters ---
-----------------------------
-- We can specify the adjustments for each character in each font using
-- the following table. All units are relative to the original width of the
-- glyph.
local horizontal_adjustments = {
["latinmodern-math.otf"] = {
["["] = {
middle = 0.6,
left_stretch = 1.0,
right_stretch = 5.0,
left_sidebearing = 0.0,
right_sidebearing = -0.75,
},
["]"] = {
middle = 0.4,
left_stretch = 5.0,
right_stretch = 1.0,
left_sidebearing = -0.5,
right_sidebearing = 0.0,
},
},
["texgyrepagella-math.otf"] = {
["("] = {
middle = 0.5,
left_stretch = 1.0,
right_stretch = 3.0,
left_sidebearing = 0.0,
right_sidebearing = -0.5,
},
[")"] = {
middle = 0.5,
left_stretch = 3.0,
right_stretch = 1.0,
left_sidebearing = -0.5,
right_sidebearing = 0.0,
},
},
["texgyrepagella-regular.otf"] = {
["r"] = {
middle = 0.6,
left_stretch = 1.0,
right_stretch = 5.0,
left_sidebearing = 0.0,
right_sidebearing = -0.7,
},
["L"] = {
middle = 0.6,
left_stretch = 1.0,
right_stretch = 5.0,
left_sidebearing = 0.0,
right_sidebearing = -0.5,
},
},
}
----------------------
--- Implementation ---
----------------------
-- Load the font-cff module from ConTeXt, which is required to use
-- "pack_result_tagged"
do
local data = io.loaddata(kpse.find_file("font-cff.lmt"))
data = data:gsub("<const>", ""):gsub("pack_result_tagged =", "fonts.handlers.otf.pack_result_tagged =")
load(data, "font-cff.lua", "t", luaotfload.fontloader)()
end
-- Cache the seen segments so that we only stretch each unique shape once.
local seen = {}
-- Stretch the glyph segments according to the provided adjustment
-- parameters.
local function stretch_glyph(segments, original_width, adjustment)
-- Extract the parameters from the adjustment table.
local middle = adjustment.middle * original_width
local left_stretch = adjustment.left_stretch
local right_stretch = adjustment.right_stretch
local left_sidebearing = adjustment.left_sidebearing
local right_sidebearing = adjustment.right_sidebearing
-- Calculate the overall scale factor
local scale_factor = (left_stretch + right_stretch) / 2
scale_factor = scale_factor + left_sidebearing + right_sidebearing
-- If we've already seen this shape, return the cached result.
if seen[segments] then
return segments, scale_factor
end
seen[segments] = true
-- Loop over each segment and stretch the x-coordinates according to
-- the provided parameters.
for _, segment in ipairs(segments) do
for i = 1, #segment - 1, 2 do
-- Get the x and y coordinates of the current point.
local x, y = segment[i], segment[i + 1]
if type(x) ~= "number" or type(y) ~= "number" then
goto continue
end
-- Stretch the x-coordinate
if x < middle then
segment[i] = left_stretch * (x - middle) + middle
else
segment[i] = right_stretch * (x - middle) + middle
end
-- Offset the x-coordinate to keep the glyph centered
segment[i] = segment[i] + (left_stretch - 1) * middle
-- Adjust the left sidebearing
segment[i] = segment[i] + (left_sidebearing * original_width)
::continue::
end
end
return segments, scale_factor
end
luatexbase.add_to_callback(
"luaotfload.patch_font",
function(tfmdata, specification, font_id)
-- Only apply the adjustments if the font filename matches one of
-- the keys in the adjustments table.
local path = tfmdata.specification.filename
local filename = file.basename(path)
local by_filename = horizontal_adjustments[filename]
if not by_filename then
return
end
-- Parameters that apply to the whole font.
tfmdata.streamprovider = 1
local size = tfmdata.size
local units_per_em = tfmdata.units_per_em
-- Loop over each character in the adjustments table for this font.
for character, adjustment in pairs(by_filename) do
-- Get the original data from the font.
local codepoint = utf8.codepoint(character)
local index = tfmdata.characters[codepoint].index
local character = tfmdata.characters[codepoint]
local shapes = fonts.hashes.shapes[font_id].glyphs
local streams = fonts.hashes.streams[font_id].streams
local original_shape = shapes[index]
-- Stretch the glyph segments according to the provided
-- adjustment parameters.
local new_segments, scale_factor = stretch_glyph(
original_shape.segments,
original_shape.width,
adjustment
)
-- Update the font data with the new segments and width.
local new_stream = fonts.handlers.otf.pack_result_tagged(
new_segments, scale_factor * original_shape.width, 0, 0
)
streams[index] = new_stream
character.width = scale_factor * character.width
end
end,
"rewrite-characters"
)
\end{luacode*}
%%%%%%%%%%%%%%%%%%%%%
%%% Demonstration %%%
%%%%%%%%%%%%%%%%%%%%%
\usepackage{fontspec}
\usepackage{unicode-math}
\pagestyle{empty}
\setlength{\parskip}{\baselineskip}
\setlength{\parindent}{0pt}
\begin{document}
\Huge
\setmathfont{Latin Modern Math}
\setmainfont{Latin Modern Roman}
$\displaystyle [a, b]^{[a, b]^{[a, b]}}$
\setmathfont{TeX Gyre Pagella Math}
\setmainfont{TeX Gyre Pagella}
$\displaystyle (a, b)_{(a, b)_{(a, b)}}$
really Long
\end{document}
