In [[ ... ]], the right hand side of a == can be a pattern:
for file in /home/directory/* ; do
if [[ $file != *mask**-mask.nii.gz ]] ; then # Skip files with mask
filename=${file##*/}
if [[ $filename == *.* ]] ; then # Is there an extension?
extension=$mask=${filename#*filename%.}
masked=${filename%"nii.$extension"gz}-mask.$extensionnii.gz
echo "$masked"
"$filename" fi"$mask"
fi
done