Skip to content
This repository was archived by the owner on Mar 1, 2026. It is now read-only.

Commit 39950d9

Browse files
inikepHerman Lee
inikep
authored and
Herman Lee
committed
Add zstd compression option for replication
Summary: This diff adds a generic protocol option to modify the compression algorithm at connect time. This is done using the 'compression_lib' connection attribute, which when combined with the compress flag, will determine a new compression library for network compression. It is meant to be extendable to other compression libraries. Protocol info: When a client wants to use compression, they can either set the compress flag as they do now, or use mysql_options to select a compression library from the given enums. Under the hood, this will set the compress flag and pass the library through to the server via a connection attribute Rollout: To change the compression on slave threads connect to the slave and run 'SET slave_compression_lib="zstd";stop slave;start slave;' This assumes the master also supports the same compression algorithms. In the event the master is not upgraded an error will occur on the slave side Reference patch: a16bf0c Differential Revision: D7142958
1 parent 815e9da commit 39950d9

15 files changed

Lines changed: 291 additions & 8 deletions

‎cmake/zstd.cmake‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@
2020
# along with this program; if not, write to the Free Software
2121
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
2222

23-
# cmake -DWITH_ZSTD=system|bundled
24-
# bundled is the default
23+
# cmake -DWITH_ZSTD=system|bundled|path_to_zstd
24+
# system is the default
2525

2626
# With earier versions, several compression tests fail.
2727
# With version < 1.0.0 our source code does not build.
@@ -77,18 +77,50 @@ ENDMACRO()
7777

7878
MACRO (MYSQL_CHECK_ZSTD)
7979
IF(NOT WITH_ZSTD)
80-
SET(WITH_ZSTD "bundled" CACHE STRING "By default use bundled zstd library")
80+
SET(WITH_ZSTD "system" CACHE STRING "By default use bundled zstd library")
8181
ENDIF()
8282

83-
IF(WITH_ZSTD STREQUAL "bundled")
83+
# See if WITH_ZSTD is of the form </path/to/custom/installation>
84+
FILE(GLOB WITH_ZSTD_HEADER ${WITH_ZSTD}/include/zstd.h)
85+
IF (WITH_ZSTD_HEADER)
86+
SET(WITH_ZSTD_PATH ${WITH_ZSTD}
87+
CACHE PATH "Path to custom ZSTD installation")
88+
ENDIF()
89+
90+
IF (WITH_ZSTD STREQUAL "bundled")
8491
MYSQL_USE_BUNDLED_ZSTD()
8592
ELSEIF(WITH_ZSTD STREQUAL "system")
8693
FIND_SYSTEM_ZSTD()
8794
IF (NOT SYSTEM_ZSTD_FOUND)
8895
MESSAGE(FATAL_ERROR "Cannot find system zstd libraries.")
8996
ENDIF()
97+
ELSEIF(WITH_ZSTD_PATH)
98+
# First search in WITH_ZSTD_PATH.
99+
FIND_PATH(ZSTD_ROOT_DIR
100+
NAMES include/zstd.h
101+
NO_CMAKE_PATH
102+
NO_CMAKE_ENVIRONMENT_PATH
103+
HINTS ${WITH_ZSTD_PATH}
104+
)
105+
106+
# Then search in standard places (if not found above).
107+
FIND_PATH(ZSTD_ROOT_DIR
108+
NAMES include/zstd.h
109+
)
110+
111+
FIND_PATH(ZSTD_INCLUDE_DIR
112+
NAMES zstd.h
113+
HINTS ${ZSTD_ROOT_DIR}/include
114+
)
115+
116+
FIND_LIBRARY(ZSTD_LIBRARY
117+
NAMES zstd_pic zstd
118+
HINTS ${ZSTD_ROOT_DIR}/lib)
119+
IF(NOT(ZSTD_INCLUDE_DIR AND ZSTD_LIBRARY))
120+
MESSAGE(FATAL_ERROR "Cannot find appropriate libraries for zstd for WITH_ZSTD=${WITH_ZSTD}.")
121+
ENDIF()
90122
ELSE()
91-
MESSAGE(FATAL_ERROR "WITH_ZSTD must be bundled or system")
123+
MESSAGE(FATAL_ERROR "Cannot find appropriate libraries for zstd.")
92124
ENDIF()
93125
FIND_ZSTD_VERSION()
94126
IF(ZSTD_VERSION VERSION_LESS MIN_ZSTD_VERSION_REQUIRED)

‎libmysql/CMakeLists.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@
2222

2323
ADD_WSHADOW_WARNING()
2424

25+
IF(NOT "$ENV{WITH_ZSTD}" STREQUAL "")
26+
SET(ZSTD_LIBRARY $ENV{WITH_ZSTD}/libzstd_pic.a)
27+
ADD_DEFINITIONS(-DZSTD)
28+
ADD_DEFINITIONS(-DHAVE_ZSTD_COMPRESS)
29+
ENDIF()
30+
2531
SET(CLIENT_API_FUNCTIONS
2632
mysql_affected_rows
2733
mysql_autocommit

‎mysql-test/r/mysqld--help-notwin.result‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2019,6 +2019,9 @@ The following options may be given as the first argument:
20192019
--slave-compressed-protocol
20202020
This option is deprecated. Use
20212021
replica_compressed_protocol instead.
2022+
--slave-compression-lib[=name]
2023+
Compression library for replication stream. 0 is zlib, 1
2024+
is zstd.
20222025
--slave-exec-mode=name
20232026
This option is deprecated. Use replica_exec_mode instead.
20242027
--slave-load-tmpdir=name
@@ -2237,6 +2240,9 @@ The following options may be given as the first argument:
22372240
same connection until the session disconnects. ON is the
22382241
only safe choice for replication.
22392242
(Defaults to on; use --skip-xa-detach-on-prepare to disable.)
2243+
--zstd-net-compression-level[=#]
2244+
Compression level for compressed protocol when zstd
2245+
library is selected. Valid values 0-22.
22402246

22412247
Variables (--variable-name=value)
22422248
abort-slave-event-count 0
@@ -2805,6 +2811,7 @@ slave-allow-batching TRUE
28052811
slave-checkpoint-group 512
28062812
slave-checkpoint-period 300
28072813
slave-compressed-protocol FALSE
2814+
slave-compression-lib zlib
28082815
slave-exec-mode STRICT
28092816
slave-max-allowed-packet 1073741824
28102817
slave-net-timeout 60
@@ -2861,6 +2868,7 @@ verbose TRUE
28612868
wait-timeout 28800
28622869
windowing-use-high-precision TRUE
28632870
xa-detach-on-prepare TRUE
2871+
zstd-net-compression-level 3
28642872

28652873
To see what values a running MySQL server is using, type
28662874
'mysqladmin variables' instead of 'mysqld --verbose --help'.

‎mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ SET PERSIST_ONLY slave_compressed_protocol = @@GLOBAL.slave_compressed_protocol;
202202
Warnings:
203203
Warning 1287 '@@slave_compressed_protocol' is deprecated and will be removed in a future release. Please use replica_compressed_protocol instead.
204204
Warning 1287 '@@slave_compressed_protocol' is deprecated and will be removed in a future release. Please use replica_compressed_protocol instead.
205+
SET PERSIST_ONLY slave_compression_lib = @@GLOBAL.slave_compression_lib;
205206
SET PERSIST_ONLY slave_exec_mode = @@GLOBAL.slave_exec_mode;
206207
Warnings:
207208
Warning 1287 '@@slave_exec_mode' is deprecated and will be removed in a future release. Please use replica_exec_mode instead.
@@ -378,6 +379,7 @@ RESET PERSIST rpl_send_buffer_size;
378379
RESET PERSIST rpl_stop_replica_timeout;
379380
RESET PERSIST session_track_gtids;
380381
RESET PERSIST skip_replica_start;
382+
RESET PERSIST slave_compression_lib;
381383
RESET PERSIST slave_rows_search_algorithms;
382384
RESET PERSIST source_verify_checksum;
383385
RESET PERSIST sql_replica_skip_counter;

‎mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,7 @@ SET PERSIST slave_compressed_protocol = @@GLOBAL.slave_compressed_protocol;
185185
Warnings:
186186
Warning 1287 '@@slave_compressed_protocol' is deprecated and will be removed in a future release. Please use replica_compressed_protocol instead.
187187
Warning 1287 '@@slave_compressed_protocol' is deprecated and will be removed in a future release. Please use replica_compressed_protocol instead.
188+
SET PERSIST slave_compression_lib = @@GLOBAL.slave_compression_lib;
188189
SET PERSIST slave_exec_mode = @@GLOBAL.slave_exec_mode;
189190
Warnings:
190191
Warning 1287 '@@slave_exec_mode' is deprecated and will be removed in a future release. Please use replica_exec_mode instead.
@@ -415,6 +416,7 @@ Warning 3615 Variable slave_checkpoint_period does not exist in persisted config
415416
RESET PERSIST IF EXISTS slave_compressed_protocol;
416417
Warnings:
417418
Warning 3615 Variable slave_compressed_protocol does not exist in persisted config file
419+
RESET PERSIST IF EXISTS slave_compression_lib;
418420
RESET PERSIST IF EXISTS slave_exec_mode;
419421
Warnings:
420422
Warning 3615 Variable slave_exec_mode does not exist in persisted config file

‎mysql-test/suite/rpl/r/rpl_test_zstd.result‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
include/master-slave.inc
2+
Warnings:
3+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
4+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
5+
[connection master]
6+
set @save_comp_lib = @@global.slave_compression_lib;
7+
set @save_slave_compress = @@global.replica_compressed_protocol;
8+
set global replica_compressed_protocol=on;
9+
include/stop_slave_io.inc
10+
include/start_slave_io.inc
11+
CREATE TABLE t1(c VARCHAR(1000));
12+
INSERT INTO t1 (c) VALUES(
13+
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
14+
'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
15+
'ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'
16+
'ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd'
17+
'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee'
18+
);
19+
include/sync_slave_sql_with_master.inc
20+
set global slave_compression_lib=zstd;
21+
include/stop_slave_io.inc
22+
include/start_slave_io.inc
23+
CREATE TABLE t2(c VARCHAR(1000));
24+
INSERT INTO t2 (c) VALUES(
25+
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
26+
'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
27+
'ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'
28+
'ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd'
29+
'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee'
30+
);
31+
include/sync_slave_sql_with_master.inc
32+
select c from t1;
33+
c
34+
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccdddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee
35+
select c from t2;
36+
c
37+
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccdddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee
38+
compression_changed
39+
1
40+
drop table t1;
41+
drop table t2;
42+
set @@global.slave_compression_lib = @save_comp_lib;
43+
set @@global.replica_compressed_protocol = @save_slave_compress;
44+
include/rpl_end.inc

‎mysql-test/suite/rpl/t/rpl_test_zstd.test‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
--source include/master-slave.inc
2+
3+
connection slave;
4+
set @save_comp_lib = @@global.slave_compression_lib;
5+
set @save_slave_compress = @@global.replica_compressed_protocol;
6+
7+
set global replica_compressed_protocol=on;
8+
9+
--source include/stop_slave_io.inc
10+
--source include/start_slave_io.inc
11+
12+
--let $zlib_before = query_get_value(show global status where variable_name='bytes_received', Value, 1)
13+
14+
connection master;
15+
16+
CREATE TABLE t1(c VARCHAR(1000));
17+
# Insert large string to ensure replication needs to compress this
18+
INSERT INTO t1 (c) VALUES(
19+
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
20+
'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
21+
'ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'
22+
'ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd'
23+
'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee'
24+
);
25+
26+
--source include/sync_slave_sql_with_master.inc
27+
28+
connection slave;
29+
--let $zlib_after = query_get_value(show global status where variable_name='bytes_received', Value, 1)
30+
31+
set global slave_compression_lib=zstd;
32+
--source include/stop_slave_io.inc
33+
--source include/start_slave_io.inc
34+
35+
--let $zstd_before = query_get_value(show global status where variable_name='bytes_received', Value, 1)
36+
37+
connection master;
38+
39+
CREATE TABLE t2(c VARCHAR(1000));
40+
# Insert large string to ensure replication needs to compress this
41+
INSERT INTO t2 (c) VALUES(
42+
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
43+
'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
44+
'ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'
45+
'ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd'
46+
'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee'
47+
);
48+
49+
--source include/sync_slave_sql_with_master.inc
50+
51+
connection slave;
52+
--let $zstd_after = query_get_value(show global status where variable_name='bytes_received', Value, 1)
53+
select c from t1;
54+
select c from t2;
55+
56+
--disable_query_log
57+
--eval select ($zlib_after - $zlib_before) <> ($zstd_after - $zstd_before) as "compression_changed"
58+
--enable_query_log
59+
60+
connection master;
61+
drop table t1;
62+
drop table t2;
63+
64+
connection slave;
65+
set @@global.slave_compression_lib = @save_comp_lib;
66+
set @@global.replica_compressed_protocol = @save_slave_compress;
67+
68+
--source include/rpl_end.inc

‎mysql-test/suite/sys_vars/r/slave_compression_lib_basic.result‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
SET @start_global_value = @@global.slave_compression_lib;
2+
SELECT @start_global_value;
3+
@start_global_value
4+
zlib
5+
SET @@global.slave_compression_lib='zlib';
6+
SELECT @@global.slave_compression_lib;
7+
@@global.slave_compression_lib
8+
zlib
9+
SET @@global.slave_compression_lib='zstd';
10+
SELECT @@global.slave_compression_lib;
11+
@@global.slave_compression_lib
12+
zstd
13+
SET @@global.slave_compression_lib=DEFAULT;
14+
SELECT @@global.slave_compression_lib;
15+
@@global.slave_compression_lib
16+
zlib
17+
SET @@global.slave_compression_lib='voodoo_compression';
18+
ERROR 42000: Variable 'slave_compression_lib' can't be set to the value of 'voodoo_compression'
19+
SELECT @@global.slave_compression_lib;
20+
@@global.slave_compression_lib
21+
zlib
22+
SET @new = @@global.slave_compression_lib;
23+
SELECT @new;
24+
@new
25+
zlib
26+
SET @@global.slave_compression_lib = @start_global_value;

‎mysql-test/suite/sys_vars/r/zstd_net_compression_level_basic.result‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
SET @orig = @@global.zstd_net_compression_level;
2+
SELECT @orig;
3+
@orig
4+
3
5+
SET @@global.zstd_net_compression_level = 2;
6+
SET @new = @@global.zstd_net_compression_level;
7+
SELECT @new;
8+
@new
9+
2
10+
SET @@global.zstd_net_compression_level = 50;
11+
Warnings:
12+
Warning 1292 Truncated incorrect zstd_net_compression_level value: '50'
13+
SELECT @@global.zstd_net_compression_level;
14+
@@global.zstd_net_compression_level
15+
22
16+
SET @@global.zstd_net_compression_level = @orig;

‎mysql-test/suite/sys_vars/t/slave_compression_lib_basic.test‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
############## mysql-test\t\slave_compression_lib_basic.test ########
2+
# #
3+
# Variable Name: slave_compression_lib #
4+
# Scope: GLOBAL #
5+
# Access Type: Dynamic #
6+
# Data Type: enum #
7+
# Default Value: 'zlib' #
8+
# Values: zlib, zstd #
9+
# Description: Which compression algorithm to use for replication #
10+
# #
11+
###############################################################################
12+
--source include/load_sysvars.inc
13+
14+
SET @start_global_value = @@global.slave_compression_lib;
15+
SELECT @start_global_value;
16+
17+
SET @@global.slave_compression_lib='zlib';
18+
SELECT @@global.slave_compression_lib;
19+
20+
SET @@global.slave_compression_lib='zstd';
21+
SELECT @@global.slave_compression_lib;
22+
23+
SET @@global.slave_compression_lib=DEFAULT;
24+
SELECT @@global.slave_compression_lib;
25+
26+
--error 1231
27+
SET @@global.slave_compression_lib='voodoo_compression';
28+
SELECT @@global.slave_compression_lib;
29+
30+
SET @new = @@global.slave_compression_lib;
31+
SELECT @new;
32+
33+
SET @@global.slave_compression_lib = @start_global_value;

0 commit comments

Comments
 (0)