WL#12168: Add Partition Information into the Binary Log

Splitting a large table into small pieces may lead to better
performance if only part of the data is accessed because less
information is retrieved or updated.

As part of this WL replication will inject into the binary log,
information on table partition (partition_id)and make it visible
through mysqlbinlog.

In particular, an external tool may need to take into account Innodb
Partitions to support change propagation on partitioned tables.

Author: Neha Kumari

libbinlogevents/include/rows_event.h

@@ -45,14 +45,21 @@
 */
 #define EXTRA_ROW_INFO_LEN_OFFSET 0
 #define EXTRA_ROW_INFO_FORMAT_OFFSET 1
-#define EXTRA_ROW_INFO_HDR_BYTES 2
-#define EXTRA_ROW_INFO_MAX_PAYLOAD (255 - EXTRA_ROW_INFO_HDR_BYTES)
+#define EXTRA_ROW_INFO_HEADER_LENGTH 2
+#define EXTRA_ROW_INFO_MAX_PAYLOAD (255 - EXTRA_ROW_INFO_HEADER_LENGTH)
 
 #define ROWS_MAPID_OFFSET 0
 #define ROWS_FLAGS_OFFSET 6
 #define ROWS_VHLEN_OFFSET 8
-#define ROWS_V_TAG_LEN 1
-#define ROWS_V_EXTRAINFO_TAG 0
+#define EXTRA_ROW_INFO_TYPECODE_LENGTH 1
+#define EXTRA_ROW_PART_INFO_VALUE_LENGTH 2
+
+/**
+  This is the typecode defined for the different elements present in
+  the container Extra_row_info, this is different from the format information
+  stored inside extra_row_ndb_info at EXTRA_ROW_INFO_FORMAT_OFFSET.
+*/
+enum class enum_extra_row_info_typecode { NDB = 0, PART = 1 };
 
 namespace binary_log {
 /**
@@ -747,9 +754,47 @@ class Table_map_event : public Binary_log_event {
   </tr>
 
   <tr>
-    <td>extra_row_data</td>
-    <td>unsigned char pointer</td>
-    <td>Pointer to extra row data if any. If non null, first byte is length</td>
+    <td>extra_row_info</td>
+    <td>An object of class Extra_row_info</td>
+    <td>The class Extra_row_info will be storing the information related
+        to m_extra_row_ndb_info and partition info (partition_id and
+        source_partition_id). At any given time a Rows_event can have both, one
+        or none of ndb_info and partition_info present as part of Rows_event.
+        In case both ndb_info and partition_info are present then below will
+        be the order in which they will be stored.
+
+        @verbatim
+        +----------+--------------------------------------+
+        |type_code |        extra_row_ndb_info            |
+        +--- ------+--------------------------------------+
+        | NDB      |Len of ndb_info |Format |ndb_data     |
+        | 1 byte   |1 byte          |1 byte |len - 2 byte |
+        +----------+----------------+-------+-------------+
+
+        In case of INSERT/DELETE
+        +-----------+----------------+
+        | type_code | partition_info |
+        +-----------+----------------+
+        |   PART    |  partition_id  |
+        | (1 byte)  |     2 byte     |
+        +-----------+----------------+
+
+        In case of UPDATE
+        +-----------+------------------------------------+
+        | type_code |        partition_info              |
+        +-----------+--------------+---------------------+
+        |   PART    | partition_id | source_partition_id |
+        | (1 byte)  |    2 byte    |       2 byte        |
+        +-----------+--------------+---------------------+
+
+        source_partition_id is used only in the case of Update_event
+        to log the partition_id of the source partition.
+
+        @endverbatim
+        This is the format for any information stored as extra_row_info.
+        type_code is not a part of the class Extra_row_info as it is a constant
+        values used at the time of serializing and decoding the event.
+   </td>
   </tr>
 
   <tr>
@@ -848,11 +893,9 @@ class Rows_event : public Binary_log_event {
       : Binary_log_event(type_arg),
         m_table_id(0),
         m_width(0),
-        m_extra_row_data(0),
         columns_before_image(0),
         columns_after_image(0),
         row(0) {}
-
   /**
     The constructor is responsible for decoding the event contained in
     the buffer.
@@ -890,13 +933,76 @@ class Rows_event : public Binary_log_event {
   uint32_t n_bits_len;   /** value determined by (m_width + 7) / 8 */
   uint16_t var_header_len;
 
-  unsigned char *m_extra_row_data;
-
   std::vector<uint8_t> columns_before_image;
   std::vector<uint8_t> columns_after_image;
   std::vector<uint8_t> row;
 
  public:
+  class Extra_row_info {
+   private:
+    /** partition_id for a row in a partitioned table */
+    int m_partition_id;
+    /**
+      It is the partition_id of the source partition in case
+      of Update_event, the target's partition_id is m_partition_id.
+      This variable is used only in case of Update_event.
+    */
+    int m_source_partition_id;
+    /** The extra row info provided by NDB */
+    unsigned char *m_extra_row_ndb_info;
+
+   public:
+    Extra_row_info()
+        : m_partition_id(UNDEFINED),
+          m_source_partition_id(UNDEFINED),
+          m_extra_row_ndb_info(nullptr) {}
+
+    Extra_row_info(const Extra_row_info &) = delete;
+
+    int get_partition_id() const { return m_partition_id; }
+    void set_partition_id(int partition_id) {
+      BAPI_ASSERT(partition_id < 65535);
+      m_partition_id = partition_id;
+    }
+
+    int get_source_partition_id() const { return m_source_partition_id; }
+    void set_source_partition_id(int source_partition_id) {
+      BAPI_ASSERT(source_partition_id < 65535);
+      m_source_partition_id = source_partition_id;
+    }
+
+    unsigned char *get_ndb_info() const { return m_extra_row_ndb_info; }
+    void set_ndb_info(const unsigned char *ndb_info, size_t len) {
+      BAPI_ASSERT(!have_ndb_info());
+      m_extra_row_ndb_info =
+          static_cast<unsigned char *>(bapi_malloc(len, 16 /* flags */));
+      std::copy(ndb_info, ndb_info + len, m_extra_row_ndb_info);
+    }
+    /**
+      Compares the extra_row_info in a Row event, it checks three things
+      1. The m_extra_row_ndb_info pointers. It compares their significant bytes.
+      2. Partition_id
+      3. source_partition_id
+
+      @return
+       true   all the above variables are same in the event and the one passed
+              in parameter.
+       false  Any of the above variable has a different value.
+    */
+    bool compare_extra_row_info(const unsigned char *ndb_info_arg,
+                                int part_id_arg, int source_part_id);
+
+    bool have_part() const { return m_partition_id != UNDEFINED; }
+
+    bool have_ndb_info() const { return m_extra_row_ndb_info != nullptr; }
+    size_t get_ndb_length();
+    size_t get_part_length();
+    ~Extra_row_info();
+
+    static const int UNDEFINED{INT_MAX};
+  };
+  Extra_row_info m_extra_row_info;
+
   unsigned long long get_table_id() const { return m_table_id.id(); }
 
   enum_flag get_flags() const { return static_cast<enum_flag>(m_flags); }

libbinlogevents/src/rows_event.cpp

@@ -361,15 +361,14 @@ Rows_event::Rows_event(const char *buf, const Format_description_event *fde)
     : Binary_log_event(&buf, fde),
       m_table_id(0),
       m_width(0),
-      m_extra_row_data(0),
       columns_before_image(0),
       columns_after_image(0),
       row(0) {
   BAPI_ENTER("Rows_event::Rows_event(const char*, ...)");
   READER_TRY_INITIALIZATION;
   READER_ASSERT_POSITION(fde->common_header_len);
   Log_event_type event_type = header()->type_code;
-  uint16_t var_header_len = 0;
+  size_t var_header_len = 0;
   size_t data_size = 0;
   uint8_t const post_header_len = fde->post_header_len[event_type - 1];
   m_type = event_type;
@@ -393,21 +392,40 @@ Rows_event::Rows_event(const char *buf, const Format_description_event *fde)
     /* Iterate over var-len header, extracting 'chunks' */
     uint64_t end = READER_CALL(position) + var_header_len;
     while (READER_CALL(position) < end) {
-      uint8_t type;
-      READER_TRY_SET(type, read<uint8_t>);
+      int type_placeholder;
+      READER_TRY_SET(type_placeholder, read<uint8_t>);
+
+      enum_extra_row_info_typecode type;
+      type = (enum_extra_row_info_typecode)type_placeholder;
       switch (type) {
-        case ROWS_V_EXTRAINFO_TAG: {
+        case enum_extra_row_info_typecode::NDB: {
           /* Have an 'extra info' section, read it in */
-          uint8_t infoLen = 0;
-          READER_TRY_SET(infoLen, read<uint8_t>);
-          /* infoLen is part of the buffer to be copied below */
+          size_t ndb_infolen = 0;
+          READER_TRY_SET(ndb_infolen, read<uint8_t>);
+          /* ndb_infolen is part of the buffer to be copied below */
           READER_CALL(go_to, READER_CALL(position) - 1);
 
           /* Just store/use the first tag of this type, skip others */
-          if (!m_extra_row_data) {
-            READER_TRY_CALL(alloc_and_memcpy, &m_extra_row_data, infoLen, 16);
+          if (!m_extra_row_info.have_ndb_info()) {
+            const char *ndb_info;
+            READER_TRY_SET(ndb_info, ptr, ndb_infolen);
+            m_extra_row_info.set_ndb_info(
+                reinterpret_cast<const unsigned char *>(ndb_info), ndb_infolen);
+            ndb_info = nullptr;
           } else {
-            READER_TRY_CALL(forward, infoLen);
+            READER_TRY_CALL(forward, ndb_infolen);
+          }
+          break;
+        }
+        case enum_extra_row_info_typecode::PART: {
+          int part_id_placeholder = 0;
+          READER_TRY_SET(part_id_placeholder, read<uint16_t>);
+          m_extra_row_info.set_partition_id(part_id_placeholder);
+          if (event_type == UPDATE_ROWS_EVENT ||
+              event_type == UPDATE_ROWS_EVENT_V1 ||
+              event_type == PARTIAL_UPDATE_ROWS_EVENT) {
+            READER_TRY_SET(part_id_placeholder, read<uint16_t>);
+            m_extra_row_info.set_source_partition_id(part_id_placeholder);
           }
           break;
         }
@@ -440,10 +458,44 @@ Rows_event::Rows_event(const char *buf, const Format_description_event *fde)
   BAPI_VOID_RETURN;
 }
 
-Rows_event::~Rows_event() {
-  if (m_extra_row_data) {
-    bapi_free(m_extra_row_data);
-    m_extra_row_data = NULL;
+Rows_event::~Rows_event() {}
+
+bool Rows_event::Extra_row_info::compare_extra_row_info(
+    const unsigned char *ndb_info_arg, int part_id_arg,
+    int source_part_id_arg) {
+  const unsigned char *ndb_row_info = m_extra_row_ndb_info;
+  bool ndb_info = ((ndb_info_arg == ndb_row_info) ||
+                   ((ndb_info_arg != NULL) && (ndb_row_info != NULL) &&
+                    (ndb_info_arg[EXTRA_ROW_INFO_LEN_OFFSET] ==
+                     ndb_row_info[EXTRA_ROW_INFO_LEN_OFFSET]) &&
+                    (memcmp(ndb_info_arg, ndb_row_info,
+                            ndb_row_info[EXTRA_ROW_INFO_LEN_OFFSET]) == 0)));
+
+  bool part_info = (part_id_arg == m_partition_id) &&
+                   (source_part_id_arg == m_source_partition_id);
+  return part_info && ndb_info;
+}
+
+size_t Rows_event::Extra_row_info::get_ndb_length() {
+  if (have_ndb_info())
+    return m_extra_row_ndb_info[EXTRA_ROW_INFO_LEN_OFFSET];
+  else
+    return 0;
+}
+
+size_t Rows_event::Extra_row_info::get_part_length() {
+  if (have_part()) {
+    if (m_source_partition_id != UNDEFINED)
+      return EXTRA_ROW_PART_INFO_VALUE_LENGTH * 2;
+    return EXTRA_ROW_PART_INFO_VALUE_LENGTH;
+  }
+  return 0;
+}
+
+Rows_event::Extra_row_info::~Extra_row_info() {
+  if (have_ndb_info()) {
+    bapi_free(m_extra_row_ndb_info);
+    m_extra_row_ndb_info = nullptr;
   }
 }
 

mysql-test/suite/rpl/include/rpl_partition_info.inc

@@ -0,0 +1,48 @@
+# ==== Purpose ====
+#
+# Assert that the binary log contains a specific sequence of
+# partition ids for INSERT and DELETE statements
+#
+# ==== Usage ====
+#
+# --let $binlog_fullpath= TEXT
+# --let $binlog_position= NUMBER
+# --let $partition_id= NUMBER
+# --source include/rpl_partition_info.inc
+#
+# Parameters:
+#
+#   $binlog_fullpath
+#     The full path of binary log to look in.
+#     (settable using include/save_binlog_position.inc)
+#   $binlog_position
+#     The log position to start reading from
+#     (settable using include/save_binlog_position.inc)
+#   $expected_partition_id
+#     The partition_id where the row changes are done. It will contain
+#     a single value or a semicolon seperated list of partition_id.
+#  ==== References ====
+#
+# Bug#28712618:INSERT/UPDATE/DELETE WITH MULTIPLE ROWS IN SAME PARTITION CREATE MULTIPLE EVENT
+# WL#12168:Add Partition Information into the Binary Log
+
+if ($expected_partition_id == '')
+{
+  --die !!!ERROR IN TEST: you must set $expected_partition_id
+}
+
+--let $include_filename= rpl_partition_info.inc
+--source include/begin_include_file.inc
+
+--exec $MYSQL_BINLOG --start-position=$binlog_position -vv $binlog_fullpath > $MYSQLTEST_VARDIR/tmp/mysqlbinlog_partition.binlog
+--let $assert_file = $MYSQLTEST_VARDIR/tmp/mysqlbinlog_partition.binlog
+
+--let $assert_select= partition:
+--let $assert_match= `SELECT CONCAT('partition: ', REPLACE('$expected_partition_id', ';', '.*\n[^\n]*partition: '))`
+--let $assert_text= assert_partition_id: $expected_partition_id
+--source include/assert_grep.inc
+
+--let $expected_partition_id=
+--remove_file $MYSQLTEST_VARDIR/tmp/mysqlbinlog_partition.binlog
+--let $include_filename= rpl_partition_info.inc
+--source include/end_include_file.inc

mysql-test/suite/rpl/include/rpl_partition_info_update.inc

@@ -0,0 +1,56 @@
+# ==== Purpose ====
+#
+# Assert that the binary log contains a specific sequence of partition ids
+# for UPDATE Events (having both a partition id for the
+# before-image($source_partition_id), and one for the after-image(target_partition_id)).
+#
+# ==== Usage ====
+#
+# --let $binlog_fullpath= TEXT
+# --let $binlog_position= NUMBER
+# --let $partition_id= NUMBER
+# --source include/rpl_partition_info.inc
+#
+# Parameters:
+#
+#   $binlog_fullpath
+#     The full path of binary log to look in.
+#     (settable using include/save_binlog_position.inc)
+#   $binlog_position
+#     The log position to start reading from
+#     (settable using include/save_binlog_position.inc)
+#   $partition_id
+#     This is a semicolon seperated list of pairs of partition_id.
+#     Each pair consists of 'source_partition_id' as an integer
+#     followed by space followed by 'target_partition_id' as an integar.
+#     Note that this is whitespace-sensitive so there should
+#     not be any other space characters than those separating
+#     source_partition_id from target_partition_id.
+#  ==== References ====
+#
+# Bug#28977199:UPDATE EVENT LOGS ONLY THE TARGET PARTITION WHEN MODIFYING MULTIPLE PARTITION
+# WL#12168:Add Partition Information into the Binary Log
+
+
+if ($partition_id == '')
+{
+  --die !!!ERROR IN TEST: you must set $target_partition_id
+}
+
+--let $include_filename= rpl_partition_info_update.inc
+--source include/begin_include_file.inc
+
+--exec $MYSQL_BINLOG --start-position=$binlog_position -vv $binlog_fullpath > $MYSQLTEST_VARDIR/tmp/mysqlbinlog_partition.binlog
+--let $assert_file = $MYSQLTEST_VARDIR/tmp/mysqlbinlog_partition.binlog
+
+--let $assert_select= source_partition:.*target_partition:.*
+--let $assert_match= `SELECT CONCAT('source_partition: ', REPLACE(REPLACE('$partition_id', ' ', '.*target_partition: '),';', '.*\n[^\n]*source_partition: '))`
+--let $assert_text= assert_partition_info: $partition_id
+--source include/assert_grep.inc
+
+--let $partition_id=
+--remove_file $MYSQLTEST_VARDIR/tmp/mysqlbinlog_partition.binlog
+--let $include_filename= rpl_partition_info_update.inc
+--source include/end_include_file.inc
+
+