\documentclass{standalone}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{fontspec}
\usepackage[math-style=ISO,bold-style=ISO,nabla=upright,partial=upright]{unicode-math}
\def\basemathfont{Latin Modern Math}
\def\myfont{Garamond Premier Pro}
\def\mymathfont{\myfont}
\def\myboldfont{ Semibold}
\def\myitalicfont{ Italic}
\setmainfont[Ligatures=TeX,
RawFeature={+calt,+onum},
ItalicFeatures={RawFeature={+calt,+dlig,+onum,+cswh,+swsh}}]{\myfont}
\let\amp\&
\renewcommand{\&}{{\fontspec{EB Garamond}\textit{\amp}}}
\setmathfont{\basemathfont}
\setmathfont[range={\mathup/{latin,Latin,greek,Greek,num},\partial,
\mathpunct,\mathbin,\mathrel,\infty,\mathpercent,\mathampersand,\mathdollar,\mathquestion},
CharacterVariant={5,11},Numbers=Lining]{\mymathfont}
\setmathfont[range={\mathit/{latin,Latin,greek,Greek,num},\ell},
CharacterVariant={5,11},Numbers=Lining]{\mymathfont \myitalicfont}
\setmathfont[range=\mathbfup/{latin,Latin,greek,Greek,num},
CharacterVariant={5,11},Numbers=Lining]{\mymathfont \myboldfont}
\setmathfont[range=\mathbfit/{latin,Latin,greek,Greek,num},
CharacterVariant={5,11},Numbers=Lining]{\mymathfont \myboldfont \myitalicfont}
\setmathfont[range={\otimes,\oplus,\square,\boxtimes,\boxminus,\boxplus,\cdot,\nabla,\ll,\gg,\lll,\ggg,\mathcolon,\Rightarrow,\Longrightarrow,\Leftrightarrow,\Longleftrightarrow}]{\basemathfont}
\setmathfont[range={\mathcal,\mathscr,\mathbfscr,
"2AA1,"2AA2,"2AF7,"2AF8,\lesssim,\gtrsim,\in,\notin,
\sumint,\equiv,\mp,\prime,\dprime,\trprime,\propto,\sim,\rightarrow,\leftarrow,\leftrightarrow,
\dots,\ddots,\vdots,\parallel,\perp}]{XITS Math}
\AtBeginDocument{\Umathcode`\* = 0 0 "002A\relax}
% Blackboard bold from AMS math
\DeclareSymbolFont{AMSb}{U}{msb}{m}{n}
\protected\def\mathbb#1{{\mathchar\numexpr256*\symAMSb+`#1\relax}}
\begin{document}
$(j)$
$(f)$
$f(x)$
$[f]$
$\{f\}$
$\displaystyle\left.\frac{\mathrm{d}}{\mathrm{d}x}f\right|_{x=4}$
\end{document}
which yields the following result:

LuaLaTeX version:

\documentclass{standalone}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{fontspec}
\usepackage[math-style=ISO,bold-style=ISO,nabla=upright,partial=upright]{unicode-math}
\def\basemathfont{Latin Modern Math}
\def\myfont{EB Garamond}
\def\mymathfont{\myfont}
\def\myitalicfont{ 12 Italic}
\setmainfont[Ligatures=TeX,
RawFeature={+calt,+onum},
ItalicFeatures={RawFeature={+calt,+dlig,+onum,+cswh,+swsh}}]{\myfont}
\let\amp\&
\renewcommand{\&}{{\fontspec{EB Garamond}\textit{\amp}}}
\setmathfont{\basemathfont}
\setmathfont[range={\mathup/{latin,Latin,greek,Greek,num},\partial,\mathpunct,\mathbin,\mathrel,\infty,\mathpercent,\mathampersand,\mathdollar,\mathquestion},CharacterVariant={5,11},Numbers=Lining]{\mymathfont}
\setmathfont[range={\mathit/{latin,Latin,greek,Greek,num},\ell},CharacterVariant={5,11},Numbers=Lining]{\mymathfont \myitalicfont}
\setmathfont[range={\otimes,\oplus,\square,\boxtimes,\boxminus,\boxplus,\cdot,\nabla,\ll,\gg,\lll,\ggg,\mathcolon,\Rightarrow,\Longrightarrow,\Leftrightarrow,\Longleftrightarrow}]{\basemathfont}
\setmathfont[range={\mathcal,\mathscr,\mathbfscr,
"2AA1,"2AA2,"2AF7,"2AF8,\lesssim,\gtrsim,\in,\notin,
\sumint,\equiv,\mp,\prime,\dprime,\trprime,\propto,\sim,\rightarrow,\leftarrow,\leftrightarrow,
\dots,\ddots,\vdots,\parallel,\perp}]{XITS Math}
\AtBeginDocument{\Umathcode`\* = 0 0 "002A\relax}
\DeclareSymbolFont{AMSb}{U}{msb}{m}{n}
\protected\def\mathbb#1{{\mathchar\numexpr256*\symAMSb+`#1\relax}}
\begin{document}
$(j)$
$(f)$
$f(x)$
$[f]$
$\{f\}$
$\displaystyle\left.\frac{\mathrm{d}}{\mathrm{d}x}f\right|_{x=4}$
\end{document}

LuaLaTeX version:

EDIT: Added LuaLaTeX versions of the examples, per comments. For this to work, one has to add \RequirePackage{luatex85} at the top and, in the case of Garamond Premier Pro, specify the font using the file names instead of the font names (this does not work with XeLaTeX though).