Merge 10.9 into 10.10

Author: dr-m

.gitlab-ci.yml

@@ -172,6 +172,39 @@ fedora-clang:
       - dependencies.dot
       - dependencies.png
 
+fedora-sanitizer:
+  stage: build
+  variables:
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: normal
+  script:
+    - yum install -y yum-utils rpm-build openssl-devel clang
+    - yum install -y /usr/lib64/libasan.so.6.0.0 /usr/lib64/libtsan.so.0.0.0 /usr/lib64/libubsan.so.1.0.0
+    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+    - yum-builddep -y mariadb-server
+    - mkdir builddir; cd builddir
+    - export CXX=${CXX:-clang++}
+    - export CC=${CC:-clang}
+    - export CXX_FOR_BUILD=${CXX_FOR_BUILD:-clang++}
+    - export CC_FOR_BUILD=${CC_FOR_BUILD:-clang}
+    - export CFLAGS='-Wno-unused-command-line-argument'
+    - export CXXFLAGS='-Wno-unused-command-line-argument'
+    - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS $SANITIZER .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: the build will fail consistently at 24% when trying to make using eatmydata
+    - make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    - *rpm_listfiles
+    - mkdir ../rpm; mv *.rpm ../rpm
+  artifacts:
+    when: always  # Must be able to see logs
+    paths:
+      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpm
+      - builddir/_CPack_Packages/Linux/RPM/SPECS/
+  parallel:
+    matrix:
+      - SANITIZER: [-DWITH_ASAN=YES, -DWITH_TSAN=YES, -DWITH_UBSAN=YES, -DWITH_MSAN=YES]
+
 centos8:
   stage: build
   image: quay.io/centos/centos:stream8 # CentOS 8 is deprecated, use this Stream8 instead
@@ -246,10 +279,8 @@ centos7:
       - rpm
       - builddir/_CPack_Packages/Linux/RPM/SPECS/
 
-mysql-test-run:
+.mysql-test-run: &mysql-test-run-def
   stage: test
-  dependencies:
-    - fedora
   script:
     # Install packages so tests and the dependencies install
     # @TODO: RPM missing 'patch' and 'diff' as dependency, so installing it manually for now
@@ -265,7 +296,76 @@ mysql-test-run:
       main.flush_logs_not_windows : query 'flush logs' succeeded - should have failed with error ER_CANT_CREATE_FILE (1004)
       main.mysql_upgrade_noengine : upgrade output order does not match the expected
       " > skiplist
-    - ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist
+    - ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist $RESTART_POLICY
+
+mysql-test-run:
+  stage: test
+  dependencies:
+    - fedora
+  <<: *mysql-test-run-def
+  artifacts:
+    when: always  # Also show results when tests fail
+    reports:
+      junit:
+        - junit.xml
+
+# Duplicate of the above jobs, except we use sanitizer build jobs as a dependency. This is so we can keep
+# sanitizer errors separate from functional test failures. Currently, there is no way to run the same
+# job for different dependencies.
+#
+# Additionally, for each sanitizer MTR job, we enable --force-restart so that 
+# sanitizer errors can be traced to individual tests. The difference in test 
+# suite runtime as a result of this flag is negligable (~30s for the entire test suite).
+# (see https://dev.mysql.com/doc/dev/mysql-server/latest/PAGE_MYSQL_TEST_RUN_PL.html)
+mysql-test-run-asan:
+  stage: test
+  variables:
+    RESTART_POLICY: "--force-restart"
+  dependencies:
+    - "fedora-sanitizer: [-DWITH_ASAN=YES]"
+  <<: *mysql-test-run-def
+  artifacts:
+    when: always  # Also show results when tests fail
+    reports:
+      junit:
+        - junit.xml
+
+mysql-test-run-tsan:
+  stage: test
+  variables:
+    RESTART_POLICY: "--force-restart"
+  dependencies:
+    - "fedora-sanitizer: [-DWITH_TSAN=YES]"
+  <<: *mysql-test-run-def
+  allow_failure: true
+  artifacts:
+    when: always  # Also show results when tests fail
+    reports:
+      junit:
+        - junit.xml
+
+mysql-test-run-ubsan:
+  stage: test
+  variables:
+    RESTART_POLICY: "--force-restart"
+  dependencies:
+    - "fedora-sanitizer: [-DWITH_UBSAN=YES]"
+  <<: *mysql-test-run-def
+  allow_failure: true
+  artifacts:
+    when: always  # Also show results when tests fail
+    reports:
+      junit:
+        - junit.xml
+
+mysql-test-run-msan:
+  stage: test
+  variables:
+    RESTART_POLICY: "--force-restart"
+  dependencies:
+    - "fedora-sanitizer: [-DWITH_MSAN=YES]"
+  <<: *mysql-test-run-def
+  allow_failure: true
   artifacts:
     when: always  # Also show results when tests fail
     reports:

CMakeLists.txt

@@ -541,7 +541,10 @@ INSTALL_DOCUMENTATION(README.md CREDITS COPYING THIRDPARTY COMPONENT Readme)
 #                      ${CMAKE_BINARY_DIR}/Docs/INFO_BIN)
 
 IF(UNIX)
-  INSTALL_DOCUMENTATION(Docs/INSTALL-BINARY Docs/README-wsrep COMPONENT Readme)
+  INSTALL_DOCUMENTATION(Docs/INSTALL-BINARY COMPONENT Readme)
+  IF(WITH_WSREP)
+    INSTALL_DOCUMENTATION(Docs/README-wsrep COMPONENT Readme)
+  ENDIF()
 ENDIF()
 
 INCLUDE(build_depends)

cmake/install_macros.cmake

@@ -265,6 +265,11 @@ SET(DEBUGBUILDDIR "${BINARY_PARENTDIR}/debug" CACHE INTERNAL "Directory of debug
 
 FUNCTION(INSTALL_MYSQL_TEST from to)
   IF(INSTALL_MYSQLTESTDIR)
+    IF(NOT WITH_WSREP)
+      SET(EXCL_GALERA "(suite/(galera|wsrep|sys_vars/[rt]/(sysvars_)?wsrep).*|include/((w.*)?wsrep.*|.*galera.*)\\.inc|std_data/(galera|wsrep).*)")
+    ELSE()
+      SET(EXCL_GALERA "^DOES_NOT_EXIST$")
+    ENDIF()
     INSTALL(
       DIRECTORY ${from}
       DESTINATION "${INSTALL_MYSQLTESTDIR}/${to}"
@@ -286,6 +291,7 @@ FUNCTION(INSTALL_MYSQL_TEST from to)
       PATTERN "*.vcxproj.user" EXCLUDE
       PATTERN "CTest*" EXCLUDE
       PATTERN "*~" EXCLUDE
+      REGEX "${EXCL_GALERA}" EXCLUDE
     )
   ENDIF()
 ENDFUNCTION()

cmake/systemd.cmake

@@ -45,7 +45,10 @@ MACRO(CHECK_SYSTEMD)
       IF(HAVE_SYSTEMD_SD_DAEMON_H AND HAVE_SYSTEMD_SD_LISTEN_FDS
          AND HAVE_SYSTEMD_SD_NOTIFY AND HAVE_SYSTEMD_SD_NOTIFYF)
         SET(HAVE_SYSTEMD TRUE)
-        SET(SYSTEMD_SCRIPTS mariadb-service-convert galera_new_cluster galera_recovery)
+        SET(SYSTEMD_SCRIPTS mariadb-service-convert)
+        IF(WITH_WSREP)
+          SET(SYSTEMD_SCRIPTS ${SYSTEMD_SCRIPTS} galera_new_cluster galera_recovery)
+        ENDIF()
         IF(DEB)
           SET(SYSTEMD_EXECSTARTPRE "ExecStartPre=/usr/bin/install -m 755 -o mysql -g root -d /var/run/mysqld")
           SET(SYSTEMD_EXECSTARTPOST "ExecStartPost=/etc/mysql/debian-start")

extra/mariabackup/backup_copy.cc

@@ -561,8 +561,8 @@ datafile_read(datafile_cur_t *cursor)
 	}
 
 	if (os_file_read(IORequestRead,
-			  cursor->file, cursor->buf, cursor->buf_offset,
-			  to_read) != DB_SUCCESS) {
+			 cursor->file, cursor->buf, cursor->buf_offset,
+			 to_read, nullptr) != DB_SUCCESS) {
 		return(XB_FIL_CUR_ERROR);
 	}
 

extra/mariabackup/changed_page_bitmap.cc

@@ -188,18 +188,15 @@ log_online_read_bitmap_page(
 {
 	ulint	checksum;
 	ulint	actual_checksum;
-	ibool	success;
 
 	ut_a(bitmap_file->size >= MODIFIED_PAGE_BLOCK_SIZE);
 	ut_a(bitmap_file->offset
 	     <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE);
 	ut_a(bitmap_file->offset % MODIFIED_PAGE_BLOCK_SIZE == 0);
-	success = os_file_read(IORequestRead,
-			       bitmap_file->file, page, bitmap_file->offset,
-			       MODIFIED_PAGE_BLOCK_SIZE) == DB_SUCCESS;
-
-	if (UNIV_UNLIKELY(!success)) {
-
+	if (DB_SUCCESS !=
+	    os_file_read(IORequestRead, bitmap_file->file, page,
+			 bitmap_file->offset, MODIFIED_PAGE_BLOCK_SIZE,
+			 nullptr)) {
 		/* The following call prints an error message */
 		os_file_get_last_error(TRUE);
 		msg("InnoDB: Warning: failed reading changed page bitmap "

extra/mariabackup/fil_cur.cc

@@ -223,7 +223,7 @@ xb_fil_cur_open(
 	if (!node->space->crypt_data
 	    && os_file_read(IORequestRead,
 			    node->handle, cursor->buf, 0,
-			    cursor->page_size) == DB_SUCCESS) {
+			    cursor->page_size, nullptr) == DB_SUCCESS) {
 		mysql_mutex_lock(&fil_system.mutex);
 		if (!node->space->crypt_data) {
 			node->space->crypt_data = fil_space_read_crypt_data(
@@ -415,12 +415,12 @@ xb_fil_cur_result_t xb_fil_cur_read(xb_fil_cur_t*	cursor,
 	cursor->buf_page_no = static_cast<unsigned>(offset / page_size);
 
 	if (os_file_read(IORequestRead, cursor->file, cursor->buf, offset,
-			  (ulint) to_read) != DB_SUCCESS) {
+			 (ulint) to_read, nullptr) != DB_SUCCESS) {
 		ret = XB_FIL_CUR_ERROR;
 		goto func_exit;
 	}
 
-	defer = space->is_deferred();
+	defer = UT_LIST_GET_FIRST(space->chain)->deferred;
 	/* check pages for corruption and re-read if necessary. i.e. in case of
 	partially written pages */
 	for (page = cursor->buf, i = 0; i < npages;

extra/mariabackup/xtrabackup.cc

@@ -3857,7 +3857,7 @@ static dberr_t xb_assign_undo_space_start()
 	byte* page = static_cast<byte*>
 		(aligned_malloc(srv_page_size, srv_page_size));
 
-	if (os_file_read(IORequestRead, file, page, 0, srv_page_size)
+	if (os_file_read(IORequestRead, file, page, 0, srv_page_size, nullptr)
 	    != DB_SUCCESS) {
 		msg("Reading first page failed.\n");
 		error = DB_ERROR;
@@ -3869,7 +3869,7 @@ static dberr_t xb_assign_undo_space_start()
 retry:
 	if (os_file_read(IORequestRead, file, page,
 			 TRX_SYS_PAGE_NO << srv_page_size_shift,
-			 srv_page_size) != DB_SUCCESS) {
+			 srv_page_size, nullptr) != DB_SUCCESS) {
 		msg("Reading TRX_SYS page failed.");
 		error = DB_ERROR;
 		goto func_exit;
@@ -5315,7 +5315,8 @@ xtrabackup_apply_delta(
 		offset = ((incremental_buffers * (page_size / 4))
 			 << page_size_shift);
 		if (os_file_read(IORequestRead, src_file,
-				 incremental_buffer, offset, page_size)
+				 incremental_buffer, offset, page_size,
+				 nullptr)
 		    != DB_SUCCESS) {
 			goto error;
 		}
@@ -5348,7 +5349,7 @@ xtrabackup_apply_delta(
 		/* read whole of the cluster */
 		if (os_file_read(IORequestRead, src_file,
 				 incremental_buffer,
-				 offset, page_in_buffer * page_size)
+				 offset, page_in_buffer * page_size, nullptr)
 		    != DB_SUCCESS) {
 			goto error;
 		}

include/CMakeLists.txt

@@ -74,7 +74,15 @@ FOREACH(f ${HEADERS_GEN_CONFIGURE})
   INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/${f}
     DESTINATION ${INSTALL_INCLUDEDIR}/server COMPONENT Development)
 ENDFOREACH(f)
-INSTALL(DIRECTORY mysql/ DESTINATION ${INSTALL_INCLUDEDIR}/server/mysql COMPONENT Development FILES_MATCHING PATTERN "*.h")
+IF(NOT WITH_WSREP)
+  SET(EXCL_SERVICE_WSREP "service_wsrep.h")
+  SET(EXCL_WSREP "wsrep.h")
+ENDIF()
+INSTALL(DIRECTORY mysql/
+        DESTINATION ${INSTALL_INCLUDEDIR}/server/mysql COMPONENT Development
+        FILES_MATCHING PATTERN "*.h"
+        PATTERN "${EXCL_SERVICE_WSREP}" EXCLUDE
+)
 
 STRING(REPLACE "." "\\." EXCL_RE "${HEADERS};${HEADERS_GEN_CONFIGURE}")
 STRING(REPLACE ";" "|" EXCL_RE "${EXCL_RE}")
@@ -85,6 +93,7 @@ MACRO(INSTALL_PRIVATE DIR)
     FILES_MATCHING PATTERN "*.h"
     PATTERN CMakeFiles EXCLUDE
     PATTERN mysql EXCLUDE
+    PATTERN "${EXCL_WSREP}" EXCLUDE
     REGEX "\\./(${EXCL_RE}$)" EXCLUDE)
 ENDMACRO()
 

man/CMakeLists.txt

@@ -12,8 +12,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1335 USA
+
 SET(MAN1_WSREP wsrep_sst_rsync.1 wsrep_sst_common.1 wsrep_sst_mariabackup.1
-    wsrep_sst_rsync_wan.1)
+    wsrep_sst_mysqldump.1 wsrep_sst_rsync_wan.1 galera_recovery.1 galera_new_cluster.1)
 SET(MAN1_SERVER innochecksum.1 myisam_ftdump.1 myisamchk.1
                 aria_chk.1 aria_dump_log.1 aria_ftdump.1 aria_pack.1 aria_read_log.1
                 aria_s3_copy.1
@@ -24,10 +25,11 @@ SET(MAN1_SERVER innochecksum.1 myisam_ftdump.1 myisamchk.1
 		mysqld_multi.1 mysqld_safe.1
 		resolveip.1 mariadb-service-convert.1
 		mysqld_safe_helper.1
-		wsrep_sst_mysqldump.1
-		galera_recovery.1 galera_new_cluster.1
 		mysql_ldb.1 myrocks_hotbackup.1
-		mbstream.1 mariabackup.1 ${MAN1_WSREP})
+		mbstream.1 mariabackup.1)
+IF(WITH_WSREP)
+  SET(MAN1_SERVER ${MAN1_SERVER} ${MAN1_WSREP})
+ENDIF()
 SET(MAN8_SERVER mysqld.8)
 SET(MAN1_CLIENT msql2mysql.1 mysql.1 mysql_find_rows.1 mysql_waitpid.1
 		mysqldumpslow.1

mysql-test/suite/encryption/r/encrypt_and_grep,undo3.rdiff

@@ -0,0 +1,32 @@
+--- encrypt_and_grep.result	2022-09-02 22:36:21.669650278 +0530
++++ encrypt_and_grep.reject	2022-11-29 19:01:22.080027528 +0530
+@@ -14,6 +14,9 @@
+ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+ NAME
+ innodb_system
++innodb_undo001
++innodb_undo002
++innodb_undo003
+ mysql/innodb_index_stats
+ mysql/innodb_table_stats
+ mysql/transaction_registry
+@@ -35,6 +38,9 @@
+ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+ NAME
+ innodb_system
++innodb_undo001
++innodb_undo002
++innodb_undo003
+ mysql/innodb_index_stats
+ mysql/innodb_table_stats
+ mysql/transaction_registry
+@@ -62,6 +68,9 @@
+ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+ NAME
+ innodb_system
++innodb_undo001
++innodb_undo002
++innodb_undo003
+ mysql/innodb_index_stats
+ mysql/innodb_table_stats
+ mysql/transaction_registry

mysql-test/suite/encryption/r/innodb-remove-encryption,undo3.rdiff

@@ -0,0 +1,32 @@
+--- innodb-remove-encryption.result	2022-09-02 20:44:59.960430396 +0530
++++ innodb-remove-encryption,undo3.reject	2022-11-29 19:02:24.813094277 +0530
+@@ -13,6 +13,9 @@
+ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+ NAME
+ innodb_system
++innodb_undo001
++innodb_undo002
++innodb_undo003
+ mysql/innodb_index_stats
+ mysql/innodb_table_stats
+ mysql/transaction_registry
+@@ -24,6 +27,9 @@
+ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+ NAME
+ innodb_system
++innodb_undo001
++innodb_undo002
++innodb_undo003
+ mysql/innodb_index_stats
+ mysql/innodb_table_stats
+ mysql/transaction_registry
+@@ -36,6 +42,9 @@
+ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+ NAME
+ innodb_system
++innodb_undo001
++innodb_undo002
++innodb_undo003
+ mysql/innodb_index_stats
+ mysql/innodb_table_stats
+ mysql/transaction_registry