WL#14683: Support openssl 3.0

RB#27023

Author: gkodinov

client/mysqltest.cc

@@ -86,6 +86,7 @@
 #include "my_dir.h"
 #include "my_inttypes.h"
 #include "my_macros.h"
+#include "my_openssl_fips.h"
 #include "my_pointer_arithmetic.h"
 #include "my_stacktrace.h"
 #include "my_systime.h"  // my_sleep()

cmake/ssl.cmake

@@ -65,6 +65,8 @@
 # pkg-config --cflags openssl11
 #    -I/usr/include/openssl11
 
+SET(MIN_OPENSSL_VERSION_REQUIRED "1.0.0")
+
 SET(WITH_SSL_DOC "\nsystem (use the OS openssl library)")
 SET(WITH_SSL_DOC "\nopenssl[0-9]+ (use alternative system library)")
 STRING_APPEND(WITH_SSL_DOC "\nyes (synonym for system)")
@@ -108,28 +110,55 @@ MACRO(RESET_SSL_VARIABLES)
   UNSET(HAVE_SHA512_DIGEST_LENGTH CACHE)
 ENDMACRO(RESET_SSL_VARIABLES)
 
+# Fetch OpenSSL version number.
+# OpenSSL < 3:
+# #define OPENSSL_VERSION_NUMBER 0x1000103fL
+# Encoded as MNNFFPPS: major minor fix patch status
+#
+# OpenSSL 3:
+# #define OPENSSL_VERSION_NUMBER
+#   ( (OPENSSL_VERSION_MAJOR<<28)
+#     |(OPENSSL_VERSION_MINOR<<20)
+#     |(OPENSSL_VERSION_PATCH<<4)
+#     |_OPENSSL_VERSION_PRE_RELEASE )
 MACRO(FIND_OPENSSL_VERSION)
-  # Verify version number. Version information looks like:
-  #   #define OPENSSL_VERSION_NUMBER 0x1000103fL
-  # Encoded as MNNFFPPS: major minor fix patch status
-  FILE(STRINGS "${OPENSSL_INCLUDE_DIR}/openssl/opensslv.h"
-    OPENSSL_VERSION_NUMBER
-    REGEX "^#[ ]*define[\t ]+OPENSSL_VERSION_NUMBER[\t ]+0x[0-9].*"
-    )
-  STRING(REGEX REPLACE
-    "^.*OPENSSL_VERSION_NUMBER[\t ]+0x([0-9]).*$" "\\1"
-    OPENSSL_MAJOR_VERSION "${OPENSSL_VERSION_NUMBER}"
-    )
-  STRING(REGEX REPLACE
-    "^.*OPENSSL_VERSION_NUMBER[\t ]+0x[0-9]([0-9][0-9]).*$" "\\1"
-    OPENSSL_MINOR_VERSION "${OPENSSL_VERSION_NUMBER}"
-    )
-  STRING(REGEX REPLACE
-    "^.*OPENSSL_VERSION_NUMBER[\t ]+0x[0-9][0-9][0-9]([0-9][0-9]).*$" "\\1"
-    OPENSSL_FIX_VERSION "${OPENSSL_VERSION_NUMBER}"
-    )
-  SET(OPENSSL_MAJOR_MINOR_FIX_VERSION "${OPENSSL_MAJOR_VERSION}")
-  STRING_APPEND(OPENSSL_MAJOR_MINOR_FIX_VERSION ".${OPENSSL_MINOR_VERSION}")
+  FOREACH(version_part
+      OPENSSL_VERSION_MAJOR
+      OPENSSL_VERSION_MINOR
+      OPENSSL_VERSION_PATCH
+      )
+    FILE(STRINGS "${OPENSSL_INCLUDE_DIR}/openssl/opensslv.h" ${version_part}
+      REGEX "^#[\t ]*define[\t ]+${version_part}[\t ]+([0-9]+).*")
+    STRING(REGEX REPLACE
+      "^.*${version_part}[\t ]+([0-9]+).*" "\\1"
+      ${version_part} "${${version_part}}")
+  ENDFOREACH()
+  IF(OPENSSL_VERSION_MAJOR VERSION_EQUAL 3)
+    # OpenSSL 3
+    SET(OPENSSL_FIX_VERSION "${OPENSSL_VERSION_PATCH}")
+  ELSE()
+    # Verify version number. Version information looks like:
+    #   #define OPENSSL_VERSION_NUMBER 0x1000103fL
+    # Encoded as MNNFFPPS: major minor fix patch status
+    FILE(STRINGS "${OPENSSL_INCLUDE_DIR}/openssl/opensslv.h"
+      OPENSSL_VERSION_NUMBER
+      REGEX "^#[ ]*define[\t ]+OPENSSL_VERSION_NUMBER[\t ]+0x[0-9].*"
+      )
+    STRING(REGEX REPLACE
+      "^.*OPENSSL_VERSION_NUMBER[\t ]+0x([0-9]).*$" "\\1"
+      OPENSSL_VERSION_MAJOR "${OPENSSL_VERSION_NUMBER}"
+      )
+    STRING(REGEX REPLACE
+      "^.*OPENSSL_VERSION_NUMBER[\t ]+0x[0-9]([0-9][0-9]).*$" "\\1"
+      OPENSSL_VERSION_MINOR "${OPENSSL_VERSION_NUMBER}"
+      )
+    STRING(REGEX REPLACE
+      "^.*OPENSSL_VERSION_NUMBER[\t ]+0x[0-9][0-9][0-9]([0-9][0-9]).*$" "\\1"
+      OPENSSL_FIX_VERSION "${OPENSSL_VERSION_NUMBER}"
+      )
+  ENDIF()
+  SET(OPENSSL_MAJOR_MINOR_FIX_VERSION "${OPENSSL_VERSION_MAJOR}")
+  STRING_APPEND(OPENSSL_MAJOR_MINOR_FIX_VERSION ".${OPENSSL_VERSION_MINOR}")
   STRING_APPEND(OPENSSL_MAJOR_MINOR_FIX_VERSION ".${OPENSSL_FIX_VERSION}")
   MESSAGE(STATUS
     "OPENSSL_VERSION (${WITH_SSL}) is ${OPENSSL_MAJOR_MINOR_FIX_VERSION}")
@@ -327,25 +356,37 @@ MACRO (MYSQL_CHECK_SSL)
         HINTS ${OPENSSL_ROOT_DIR}/include
       )
       MESSAGE(STATUS "OPENSSL_APPLINK_C ${OPENSSL_APPLINK_C}")
+      IF(NOT OPENSSL_APPLINK_C)
+        RESET_SSL_VARIABLES()
+        FATAL_SSL_NOT_FOUND_ERROR(
+          "Cannot find applink.c for WITH_SSL=${WITH_SSL}.")
+      ENDIF()
     ENDIF()
 
     FIND_LIBRARY(OPENSSL_LIBRARY
                  NAMES ssl libssl ssleay32 ssleay32MD
-                 HINTS ${OPENSSL_ROOT_DIR}/lib)
+                 HINTS ${OPENSSL_ROOT_DIR}/lib ${OPENSSL_ROOT_DIR}/lib64)
     FIND_LIBRARY(CRYPTO_LIBRARY
                  NAMES crypto libcrypto libeay32
-                 HINTS ${OPENSSL_ROOT_DIR}/lib)
+                 HINTS ${OPENSSL_ROOT_DIR}/lib ${OPENSSL_ROOT_DIR}/lib64)
 
     IF(OPENSSL_INCLUDE_DIR)
       FIND_OPENSSL_VERSION()
     ENDIF()
+    IF (OPENSSL_MAJOR_MINOR_FIX_VERSION VERSION_LESS
+        ${MIN_OPENSSL_VERSION_REQUIRED})
+      RESET_SSL_VARIABLES()
+      FATAL_SSL_NOT_FOUND_ERROR(
+        "Not a supported openssl version in WITH_SSL=${WITH_SSL}.")
+    ENDIF()
+
     IF("${OPENSSL_MAJOR_MINOR_FIX_VERSION}" VERSION_GREATER "1.1.0")
        ADD_DEFINITIONS(-DHAVE_TLSv13)
     ENDIF()
+
     IF(OPENSSL_INCLUDE_DIR AND
-       OPENSSL_LIBRARY   AND
-       CRYPTO_LIBRARY      AND
-       OPENSSL_MAJOR_VERSION STREQUAL "1"
+       OPENSSL_LIBRARY     AND
+       CRYPTO_LIBRARY
       )
       SET(OPENSSL_FOUND TRUE)
       IF(WITH_SSL_PATH)
@@ -412,8 +453,8 @@ MACRO (MYSQL_CHECK_SSL)
     MESSAGE(STATUS "OPENSSL_LIBRARY = ${OPENSSL_LIBRARY}")
     MESSAGE(STATUS "CRYPTO_LIBRARY = ${CRYPTO_LIBRARY}")
     MESSAGE(STATUS "OPENSSL_LIB_DIR = ${OPENSSL_LIB_DIR}")
-    MESSAGE(STATUS "OPENSSL_MAJOR_VERSION = ${OPENSSL_MAJOR_VERSION}")
-    MESSAGE(STATUS "OPENSSL_MINOR_VERSION = ${OPENSSL_MINOR_VERSION}")
+    MESSAGE(STATUS "OPENSSL_VERSION_MAJOR = ${OPENSSL_VERSION_MAJOR}")
+    MESSAGE(STATUS "OPENSSL_VERSION_MINOR = ${OPENSSL_VERSION_MINOR}")
     MESSAGE(STATUS "OPENSSL_FIX_VERSION = ${OPENSSL_FIX_VERSION}")
 
     INCLUDE(CheckSymbolExists)
@@ -669,15 +710,21 @@ MACRO(MYSQL_CHECK_SSL_DLLS)
       GET_FILENAME_COMPONENT(OPENSSL_NAME "${OPENSSL_LIBRARY}" NAME_WE)
 
       # Different naming scheme for the matching .dll as of SSL 1.1
+      # OpenSSL 3.x Look for libcrypto-3-x64.dll or libcrypto-3.dll
+      # OpenSSL 1.1 Look for libcrypto-1_1-x64.dll or libcrypto-1_1.dll
+      # OpenSSL 1.0 Look for libeay32.dll
       SET(SSL_MSVC_VERSION_SUFFIX)
       SET(SSL_MSVC_ARCH_SUFFIX)
-      IF(OPENSSL_MINOR_VERSION VERSION_EQUAL 1)
+      IF(OPENSSL_VERSION_MAJOR VERSION_EQUAL 1 AND
+         OPENSSL_VERSION_MINOR VERSION_EQUAL 1)
         SET(SSL_MSVC_VERSION_SUFFIX "-1_1")
         SET(SSL_MSVC_ARCH_SUFFIX "-x64")
       ENDIF()
+      IF(OPENSSL_VERSION_MAJOR VERSION_EQUAL 3)
+        SET(SSL_MSVC_VERSION_SUFFIX "-3")
+        SET(SSL_MSVC_ARCH_SUFFIX "-x64")
+      ENDIF()
 
-      # OpenSSL 1.1 Look for libcrypto-1_1-x64.dll or libcrypto-1_1.dll
-      # OpenSSL 1.0 Look for libeay32.dll
       FIND_FILE(HAVE_CRYPTO_DLL
         NAMES
         "${CRYPTO_NAME}${SSL_MSVC_VERSION_SUFFIX}${SSL_MSVC_ARCH_SUFFIX}.dll"
@@ -717,11 +764,25 @@ MACRO(MYSQL_CHECK_SSL_DLLS)
         ADD_DEPENDENCIES(${openssl_exe_target} copy_openssl_dlls)
       ELSE()
         MESSAGE(STATUS "Cannot find SSL dynamic libraries")
-        IF(OPENSSL_MINOR_VERSION VERSION_EQUAL 1)
+        IF(OPENSSL_VERSION_MAJOR VERSION_EQUAL 1 AND
+           OPENSSL_VERSION_MINOR VERSION_EQUAL 1)
           SET(SSL_LIBRARIES ${SSL_LIBRARIES} crypt32.lib)
           MESSAGE(STATUS "SSL_LIBRARIES ${SSL_LIBRARIES}")
         ENDIF()
       ENDIF()
     ENDIF()
   ENDIF()
 ENDMACRO()
+
+# Downgrade OpenSSL 3 deprecation warnings.
+MACRO(DOWNGRADE_OPENSSL3_DEPRECATION_WARNINGS)
+  IF(OPENSSL_VERSION_MAJOR VERSION_EQUAL 3)
+    IF(MY_COMPILER_IS_GNU_OR_CLANG)
+      ADD_COMPILE_FLAGS(${ARGV}
+        COMPILE_FLAGS "-Wno-error=deprecated-declarations")
+    ELSEIF(WIN32)
+      ADD_COMPILE_FLAGS(${ARGV}
+        COMPILE_FLAGS "/wd4996")
+    ENDIF()
+  ENDIF()
+ENDMACRO()

extra/libfido2/libfido2-1.8.0/src/CMakeLists.txt

@@ -85,6 +85,52 @@ list(APPEND COMPAT_SOURCES
 	../openbsd-compat/timingsafe_bcmp.c
 )
 
+DOWNGRADE_OPENSSL3_DEPRECATION_WARNINGS(
+  assert.c
+  cbor.c
+  cred.c
+  es256.c
+  rs256.c
+  ecdh.c
+  )
+
+IF(OPENSSL_VERSION_MAJOR VERSION_EQUAL 3)
+  IF(MY_COMPILER_IS_GNU)
+    # Downgrade to warning, for now.
+    ADD_COMPILE_FLAGS(
+      assert.c
+      cbor.c
+      cred.c
+      es256.c
+      rs256.c
+      COMPILE_FLAGS "-Wno-error=discarded-qualifiers")
+    ADD_COMPILE_FLAGS(
+      ecdh.c
+      COMPILE_FLAGS "-Wno-error=pointer-sign")
+  ELSEIF(MY_COMPILER_IS_CLANG)
+    # Downgrade to warning, for now.
+    ADD_COMPILE_FLAGS(
+      assert.c
+      cbor.c
+      cred.c
+      es256.c
+      rs256.c
+      COMPILE_FLAGS
+      "-Wno-incompatible-pointer-types-discards-qualifiers")
+    ADD_COMPILE_FLAGS(
+      ecdh.c
+      COMPILE_FLAGS "-Wno-pointer-sign")
+  ELSEIF(MSVC)
+    ADD_COMPILE_FLAGS(
+      assert.c
+      cred.c
+      COMPILE_FLAGS "/wd4090")
+    ADD_COMPILE_FLAGS(
+      ecdh.c
+      COMPILE_FLAGS "/wd4057")
+  ENDIF()
+ENDIF()
+
 
 # Windows wants only major.minor
 IF(WIN32)

include/my_openssl_fips.h

@@ -0,0 +1,41 @@
+/* Copyright (c) 2022, Oracle and/or its affiliates.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License, version 2.0,
+   as published by the Free Software Foundation.
+
+   This program is also distributed with certain software (including
+   but not limited to OpenSSL) that is licensed under separate terms,
+   as designated in a particular file or component or in included license
+   documentation.  The authors of MySQL hereby grant you an additional
+   permission to link the program and your derivative works with the
+   separately licensed software that they have included with MySQL.
+
+   Without limiting anything contained in the foregoing, this file,
+   which is part of C Driver for MySQL (Connector/C), is also subject to the
+   Universal FOSS Exception, version 1.0, a copy of which can be found at
+   http://oss.oracle.com/licenses/universal-foss-exception.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License, version 2.0, for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#ifndef my_openssl_fips_h_
+#define my_openssl_fips_h_
+
+constexpr int OPENSSL_ERROR_LENGTH{512}; /* Openssl error code max length */
+
+bool set_fips_mode(const int fips_mode, char err_string[OPENSSL_ERROR_LENGTH]);
+int get_fips_mode();
+
+int test_ssl_fips_mode(char err_string[OPENSSL_ERROR_LENGTH]);
+
+void fips_deinit();
+void fips_init();
+
+#endif /* ifndef my_openssl_fips_h_ */

include/violite.h

@@ -140,7 +140,6 @@ enum enum_vio_io_event {
 #define VIO_LOCALHOST 1            /* a localhost connection */
 #define VIO_BUFFERED_READ 2        /* use buffered read */
 #define VIO_READ_BUFFER_SIZE 16384 /* size of read buffer */
-#define OPENSSL_ERROR_LENGTH 512   /* Openssl error code max length */
 
 MYSQL_VIO vio_new(my_socket sd, enum enum_vio_type type, uint flags);
 MYSQL_VIO mysql_socket_vio_new(MYSQL_SOCKET mysql_socket,
@@ -268,12 +267,6 @@ struct st_VioSSLFd *new_VioSSLConnectorFd(
 
 long process_tls_version(const char *tls_version);
 
-int set_fips_mode(const uint fips_mode, char *err_string);
-
-uint get_fips_mode();
-
-int test_ssl_fips_mode(char *err_string);
-
 struct st_VioSSLFd *new_VioSSLAcceptorFd(
     const char *key_file, const char *cert_file, const char *ca_file,
     const char *ca_path, const char *cipher, const char *ciphersuites,

libmysql/authentication_oci_client/CMakeLists.txt

@@ -52,3 +52,7 @@ MYSQL_ADD_PLUGIN(
   CLIENT_ONLY
   MODULE_ONLY MODULE_OUTPUT_NAME "authentication_oci_client"
 )
+
+DOWNGRADE_OPENSSL3_DEPRECATION_WARNINGS(
+  src/oci_iam/request/ssl.cc
+  )

mysql-test/suite/auth_sec/r/openssl_without_fips.result

@@ -17,7 +17,7 @@ c9f0f895fb98ab9159f51fd0297e236d
 # Bug #33082255: SERVER EXIT TRYING TO SET FIPS MODE
 #
 SET @@global.ssl_fips_mode = 'ON';
-ERROR HY000: SSL fips mode error: Openssl is not fips enabled
+ERROR HY000: SSL fips mode error: Openssl is not fips enabled: openssl error
 ##Test: Start the server with SSL FIPS mode ON, server will throw error and abort.
-Pattern "(FIPS_mode_set:fips mode not supported|FIPS_module_mode_set:fingerprint does not match)" found
+Pattern "(FIPS_mode_set:fips mode not supported|FIPS_module_mode_set:fingerprint does not match|SSL fips mode error:)" found
 Restart server with FIPS mode OFF.

mysql-test/suite/auth_sec/t/openssl_without_fips.test

@@ -51,6 +51,7 @@ EOF
 --echo # Bug #33082255: SERVER EXIT TRYING TO SET FIPS MODE
 --echo #
 
+--replace_regex /SSL fips mode error: Openssl is not fips enabled.*/SSL fips mode error: Openssl is not fips enabled: openssl error/
 --error ER_DA_SSL_FIPS_MODE_ERROR
 SET @@global.ssl_fips_mode = 'ON';
 
@@ -65,7 +66,7 @@ let $restart_file = $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
 --source include/wait_until_disconnected.inc
 --error 1
 --exec $MYSQLD_CMD --loose-console --ssl-fips-mode=ON > $error_log 2>&1
-let SEARCH_PATTERN = (FIPS_mode_set:fips mode not supported|FIPS_module_mode_set:fingerprint does not match);
+let SEARCH_PATTERN = (FIPS_mode_set:fips mode not supported|FIPS_module_mode_set:fingerprint does not match|SSL fips mode error:);
 --source include/search_pattern.inc
 --remove_file $error_log
 

mysys/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2006, 2021, Oracle and/or its affiliates.
+# Copyright (c) 2006, 2022, Oracle and/or its affiliates.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License, version 2.0,
@@ -119,6 +119,7 @@ SET(MYSYS_SOURCES
   my_sha2.cc
   my_md5.cc
   my_rnd.cc
+  my_openssl_fips.cc
 )
 LIST(APPEND MYSYS_SOURCES my_aes_openssl.cc)