I might:

• Use match rather than isinstance.
• Put the list of levels into a module constant
• Factor out the common error message as a variable.
• Break the very long line.
• Import Optional from typing to replace hints like str | None.

import logging
from typing import Optional

LEVEL_STRINGS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
LEVEL_INTS = [10, 20, 30, 40, 50]

def logger(
    name = __name__, 
    level: int | str = logging.INFO, 
    filename: Optional[str] = None
) -> logging.Logger:
    err_msg = "Некорректный уровень логирования"

    match level:
        case str():
            if level not in LEVEL_STRINGS:
                raise ValueError(err_msg)

            level = logging.getLevelName(level.upper())
        case int():
            if level not in LEVEL_INTS:
                raise ValueError(err_msg)
        case _:
            raise TypeError(f"Неккоректный тип данных уровня логирования")

    if not logging.getLogger(name).handlers:
        format_log = "%(asctime)s | %(levelname)s | %(name)s:%(module)s:%(lineno)s - %(message)s"
        logging.basicConfig(level=level, format=format_log, filename=filename)

    return logging.getLogger(name)

if __name__ == "__main__":
    logger().info("Логер успешно запущен")

You should also incorporate docstrings for your function and for the module, noting among other things, accepted values for level.

I might:

• Use match rather than conditionals testing with isinstance.
• Put the list of levels into a module constant
• Factor out the common error message as a variable.
• Break the very long line.
• Import Optional from typing to replace hints like str | None.

import logging
from typing import Optional

LEVEL_STRINGS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
LEVEL_INTS = [10, 20, 30, 40, 50]

def logger(
    name = __name__, 
    level: int | str = logging.INFO, 
    filename: Optional[str] = None
) -> logging.Logger:
    value_err_msg = "Некорректный уровень логирования"

    match level:
        case str():
            if level not in LEVEL_STRINGS:
                raise ValueError(value_err_msg)

            level = logging.getLevelName(level.upper())
        case int():
            if level not in LEVEL_INTS:
                raise ValueError(value_err_msg)
        case _:
            raise TypeError(f"Неккоректный тип данных уровня логирования")

    if not logging.getLogger(name).handlers:
        format_log = "%(asctime)s | %(levelname)s | %(name)s:%(module)s:%(lineno)s - %(message)s"
        logging.basicConfig(level=level, format=format_log, filename=filename)

    return logging.getLogger(name)

if __name__ == "__main__":
    logger().info("Логер успешно запущен")

You should also incorporate docstrings for your function and for the module, noting among other things, accepted values for level.

When level is a string, you've already ensured it must be within a set of all uppercase words, so why are you then calling level.upper()? Doesn't this seem redundant?