diff --git a/doc/dev_guide/internals/iproto/images/repl_ballot.puml b/doc/dev_guide/internals/iproto/images/repl_ballot.puml index 3065040e02..964a50391e 100644 --- a/doc/dev_guide/internals/iproto/images/repl_ballot.puml +++ b/doc/dev_guide/internals/iproto/images/repl_ballot.puml @@ -8,13 +8,15 @@ skinparam { } json "**IPROTO_BALLOT body**" as ballot { - "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys IPROTO_BALLOT_IS_RO_CFG]]": "MP_BOOL", - "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys IPROTO_BALLOT_VCLOCK]]": "MP_MAP", - "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys IPROTO_BALLOT_GC_VCLOCK]]": "MP_MAP", - "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys IPROTO_BALLOT_IS_RO]]": "MP_BOOL", - "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys IPROTO_BALLOT_IS_ANON]]": "MP_BOOL", - "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys IPROTO_BALLOT_IS_BOOTED]]": "MP_BOOL", - "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys IPROTO_BALLOT_CAN_LEAD]]": "MP_BOOL" + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general-replication IPROTO_BALLOT_IS_RO_CFG]]": "MP_BOOL", + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general-replication IPROTO_BALLOT_VCLOCK]]": "MP_MAP", + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general-replication IPROTO_BALLOT_GC_VCLOCK]]": "MP_MAP", + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general-replication IPROTO_BALLOT_IS_RO]]": "MP_BOOL", + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general-replication IPROTO_BALLOT_IS_ANON]]": "MP_BOOL", + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general-replication IPROTO_BALLOT_IS_BOOTED]]": "MP_BOOL", + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general-replication IPROTO_BALLOT_CAN_LEAD]]": "MP_BOOL", + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general-replication IPROTO_BALLOT_BOOTSTRAP_LEADER_UUID]]": "MP_STR", + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general-replication IPROTO_BALLOT_REGISTERED_REPLICA_UUIDS]]": "MP_ARRAY" } @enduml diff --git a/doc/dev_guide/internals/iproto/images/repl_ballot.svg b/doc/dev_guide/internals/iproto/images/repl_ballot.svg index 6cb71c9499..6ae1ea5aa5 100644 --- a/doc/dev_guide/internals/iproto/images/repl_ballot.svg +++ b/doc/dev_guide/internals/iproto/images/repl_ballot.svg @@ -1,30 +1 @@ -IPROTO_BALLOT bodyIPROTO_BALLOT_IS_RO_CFGMP_BOOLIPROTO_BALLOT_VCLOCKMP_MAPIPROTO_BALLOT_GC_VCLOCKMP_MAPIPROTO_BALLOT_IS_ROMP_BOOLIPROTO_BALLOT_IS_ANONMP_BOOLIPROTO_BALLOT_IS_BOOTEDMP_BOOLIPROTO_BALLOT_CAN_LEADMP_BOOL \ No newline at end of file +IPROTO_BALLOT bodyIPROTO_BALLOT_IS_RO_CFGMP_BOOLIPROTO_BALLOT_VCLOCKMP_MAPIPROTO_BALLOT_GC_VCLOCKMP_MAPIPROTO_BALLOT_IS_ROMP_BOOLIPROTO_BALLOT_IS_ANONMP_BOOLIPROTO_BALLOT_IS_BOOTEDMP_BOOLIPROTO_BALLOT_CAN_LEADMP_BOOLIPROTO_BALLOT_BOOTSTRAP_LEADER_UUIDMP_STRIPROTO_BALLOT_REGISTERED_REPLICA_UUIDSMP_ARRAY \ No newline at end of file diff --git a/doc/dev_guide/internals/iproto/images/repl_heartbeat.svg b/doc/dev_guide/internals/iproto/images/repl_heartbeat.svg deleted file mode 100644 index a6c28bc70f..0000000000 --- a/doc/dev_guide/internals/iproto/images/repl_heartbeat.svg +++ /dev/null @@ -1,41 +0,0 @@ -Heartbeat message from masterSizeMP_UINTHeaderIPROTO_REQUEST_TYPEIPROTO_OKIPROTO_REPLICA_IDMP_UINTIPROTO_TIMESTAMPMP_DOUBLEResponse from replicaSizeMP_UINTHeaderIPROTO_REQUEST_TYPEIPROTO_OKIPROTO_REPLICA_IDMP_UINT(Optional)IPROTO_TIMESTAMPMP_DOUBLEBodyIPROTO_VCLOCKMP_MAP \ No newline at end of file diff --git a/doc/dev_guide/internals/iproto/images/repl_heartbeat_message.puml b/doc/dev_guide/internals/iproto/images/repl_heartbeat_message.puml new file mode 100644 index 0000000000..c5adefe8ab --- /dev/null +++ b/doc/dev_guide/internals/iproto/images/repl_heartbeat_message.puml @@ -0,0 +1,22 @@ +@startuml + +skinparam { + HyperlinkColor #0077FF + FontColor #313131 + BorderColor #313131 + BackgroundColor transparent +} + +json "**Heartbeat message from master**" as heartbeat_request { + "Size": "MP_UINT", + "Header": { + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general IPROTO_REQUEST_TYPE]]": "IPROTO_OK", + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general-replication IPROTO_REPLICA_ID]]": "MP_UINT", + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general IPROTO_TIMESTAMP]]": "MP_DOUBLE" + }, + "Body": { + "(Optional) [[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general-replication IPROTO_VCLOCK_SYNC]]": "MP_UINT" + } +} + +@enduml diff --git a/doc/dev_guide/internals/iproto/images/repl_heartbeat_message.svg b/doc/dev_guide/internals/iproto/images/repl_heartbeat_message.svg new file mode 100644 index 0000000000..3d6a9cbbde --- /dev/null +++ b/doc/dev_guide/internals/iproto/images/repl_heartbeat_message.svg @@ -0,0 +1 @@ +Heartbeat message from masterSizeMP_UINTHeaderIPROTO_REQUEST_TYPEIPROTO_OKIPROTO_REPLICA_IDMP_UINTIPROTO_TIMESTAMPMP_DOUBLEBody(Optional)IPROTO_VCLOCK_SYNCMP_UINT \ No newline at end of file diff --git a/doc/dev_guide/internals/iproto/images/repl_heartbeat.puml b/doc/dev_guide/internals/iproto/images/repl_heartbeat_response.puml similarity index 50% rename from doc/dev_guide/internals/iproto/images/repl_heartbeat.puml rename to doc/dev_guide/internals/iproto/images/repl_heartbeat_response.puml index e52ac27047..39dc816edf 100644 --- a/doc/dev_guide/internals/iproto/images/repl_heartbeat.puml +++ b/doc/dev_guide/internals/iproto/images/repl_heartbeat_response.puml @@ -7,24 +7,17 @@ skinparam { BackgroundColor transparent } -json "**Heartbeat message from master**" as heartbeat_request { - "Size": "MP_UINT", - "Header": { - "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys IPROTO_REQUEST_TYPE]]": "IPROTO_OK", - "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys IPROTO_REPLICA_ID]]": "MP_UINT", - "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys IPROTO_TIMESTAMP]]": "MP_DOUBLE" - } -} - json "**Response from replica**" as heartbeat_response { "Size": "MP_UINT", "Header": { - "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys IPROTO_REQUEST_TYPE]]": "IPROTO_OK", - "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys IPROTO_REPLICA_ID]]": "MP_UINT", - "(Optional) [[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys IPROTO_TIMESTAMP]]": "MP_DOUBLE" + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/key/#general IPROTO_REQUEST_TYPE]]": "IPROTO_OK", + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general-replication IPROTO_REPLICA_ID]]": "MP_UINT", + "(Optional) [[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general IPROTO_TIMESTAMP]]": "MP_DOUBLE" }, "Body": { - "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys IPROTO_VCLOCK]]": "MP_MAP" + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general-replication IPROTO_VCLOCK]]": "MP_MAP", + "(Optional) [[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#general-replication IPROTO_VCLOCK_SYNC]]": "MP_UINT", + "[[https://tarantool.io/en/doc/latest/dev_guide/internals/iproto/keys/#synchronous-replication IPROTO_TERM]]": "MP_UINT" } } diff --git a/doc/dev_guide/internals/iproto/images/repl_heartbeat_response.svg b/doc/dev_guide/internals/iproto/images/repl_heartbeat_response.svg new file mode 100644 index 0000000000..e4f15d9673 --- /dev/null +++ b/doc/dev_guide/internals/iproto/images/repl_heartbeat_response.svg @@ -0,0 +1 @@ +Response from replicaSizeMP_UINTHeaderIPROTO_REQUEST_TYPEIPROTO_OKIPROTO_REPLICA_IDMP_UINT(Optional)IPROTO_TIMESTAMPMP_DOUBLEBodyIPROTO_VCLOCKMP_MAP(Optional)IPROTO_VCLOCK_SYNCMP_UINTIPROTO_TERMMP_UINT \ No newline at end of file diff --git a/doc/dev_guide/internals/iproto/keys.rst b/doc/dev_guide/internals/iproto/keys.rst index 39858ee18f..5c3ccc12d9 100644 --- a/doc/dev_guide/internals/iproto/keys.rst +++ b/doc/dev_guide/internals/iproto/keys.rst @@ -185,9 +185,15 @@ General replication - 0x26 |br| MP_MAP - The instance's vclock - * - IPROTO_CLUSTER_UUID + * - :ref:`IPROTO_VCLOCK_SYNC ` + - 0x5a |br| MP_UINT + - ID of the vclock synchronization request. + Since 2.11 + + * - IPROTO_REPLICASET_UUID - 0x25 |br| MP_STR - - Cluster UUID + - UUID of the replica set. + Prior to Tarantool version 2.11, IPROTO_REPLICASET_UUID was called IPROTO_CLUSTER_UUID. * - IPROTO_LSN - 0x03 |br| MP_UINT @@ -197,7 +203,7 @@ General replication - 0x08 |br| MP_UINT - Transaction sequence number - * - IPROTO_BALLOT_IS_RO_CFG + * - :ref:`IPROTO_BALLOT_IS_RO_CFG ` - 0x01 |br| MP_BOOL - True if the instance is configured as :ref:`read_only `. Since :doc:`2.6.1 ` @@ -210,28 +216,39 @@ General replication - 0x03 |br| MP_MAP - Vclock of the instance’s oldest WAL entry - * - IPROTO_BALLOT_IS_RO + * - :ref:`IPROTO_BALLOT_IS_RO ` - 0x04 |br| MP_BOOL - True if the instance is not writable: configured as :ref:`read_only `, has :ref:`orphan status `, or is a :ref:`Raft follower `. Since :doc:`2.6.1 ` - * - IPROTO_BALLOT_IS_ANON + * - :ref:`IPROTO_BALLOT_IS_ANON ` - 0x05 |br| MP_BOOL - True if the replica is anonymous. Corresponds to :ref:`box.cfg.replication_anon `. Since :doc:`2.7.1 ` - * - IPROTO_BALLOT_IS_BOOTED + * - :ref:`IPROTO_BALLOT_IS_BOOTED ` - 0x06 |br| MP_BOOL - True if the instance has finished its bootstrap or recovery process. Since :doc:`2.7.3 `, :doc:`2.8.2 `, :doc:`2.10.0 ` - * - IPROTO_BALLOT_CAN_LEAD + * - :ref:`IPROTO_BALLOT_CAN_LEAD ` - 0x07 |br| MP_BOOL - True if :ref:`box.cfg.election_mode ` is ``candidate`` or ``manual``. Since v. :doc:`2.7.3 ` and :doc:`2.8.2 ` + + * - :ref:`IPROTO_BALLOT_BOOTSTRAP_LEADER_UUID ` + - 0x08 |br| MP_STR + - UUID of the bootstrap leader. The UUID is encoded as a 36-byte string. + Since v. 2.11 + + * - :ref:`IPROTO_BALLOT_REGISTERED_REPLICA_UUIDS ` + - 0x09 |br| MP_ARRAY + - An array of MP_STR elements that contains the UUIDs of members registered in the replica set. + Each UUID is encoded as a 36-byte string. + Since v. 2.11 * - :ref:`IPROTO_FLAGS ` - 0x09 |br| MP_UINT @@ -254,12 +271,7 @@ General replication followed by an array of ids of instances whose rows won't be relayed to the replica. Since v. :doc:`2.10.0 ` -All IPROTO_BALLOT_* keys are only used in :ref:`IPROTO_BALLOT ` requests. -There have been some name changes starting with versions 2.7.3, 2.8.2, and 2.10.0: - -* IPROTO_BALLOT_IS_RO_CFG was formerly called IPROTO_BALLOT_IS_RO. -* IPROTO_BALLOT_IS_RO was formerly called IPROTO_BALLOT_IS_LOADING. - +.. _internals-iproto-keys-syncro-replication: Synchronous replication ~~~~~~~~~~~~~~~~~~~~~~~ @@ -274,14 +286,14 @@ Synchronous replication - Code and |br| value type - Description - * - IPROTO_TERM + * - :ref:`IPROTO_TERM ` - 0x53 |br| MP_UINT - - The term in which the node becomes the :ref:`synchronous queue owner `. - Used in :ref:`IPROTO_RAFT_PROMOTE ` + - :ref:`RAFT term ` on an instance * - IPROTO_RAFT_TERM - 0x00 |br| MP_UINT - - :ref:`RAFT term ` on an instance + - :ref:`RAFT term ` on an instance. + The key is only used for requests with the :ref:`IPROTO_RAFT type`. * - IPROTO_RAFT_VOTE - 0x01 |br| MP_UINT @@ -289,7 +301,7 @@ Synchronous replication * - IPROTO_RAFT_STATE - 0x02 |br| MP_UINT - - RAFT state. Possible values: ``1`` -- follower, ``2`` -- candidate, ``3`` -- leader + - :ref:`RAFT state `. Possible values: ``1`` -- follower, ``2`` -- candidate, ``3`` -- leader * - :ref:`IPROTO_RAFT_VCLOCK ` - 0x03 |br| MP_MAP @@ -652,6 +664,19 @@ Example: .. raw:: html :file: images/flags_example.svg +.. _internals-iproto-keys-term: + +IPROTO_TERM +~~~~~~~~~~~ + +Code: 0x53. + +* The key is used in the :ref:`IPROTO_RAFT_PROMOTE ` + and :ref:`IPROTO_RAFT_DEMOTE ` requests. + +* Since version 2.11, the key is included in response to a :ref:`heartbeat message `. + The term corresponds to the value of :ref:`box.info.synchro.queue.term ` on the sender instance. + .. _internals-iproto-keys-vclock: Vclock keys @@ -663,15 +688,23 @@ In fact, it represents the number of logical operations executed on a specific n .. raw:: html :file: images/vclock.svg -There are four keys that correspond to vector clocks in different contexts of replication. +There are five keys that correspond to vector clocks in different contexts of replication. They all have the MP_MAP type: * IPROTO_VCLOCK (0x26) is passed to a new instance :ref:`joining the replica set `. -* IPROTO_BALLOT_VCLOCK (0x02) is sent in response to :ref:`IPROTO_VOTE `. +* IPROTO_VCLOCK_SYNC (0x5a) is used by :ref:`replication heartbeats `. + The master sends its heartbeats, including this monotonically growing key, to a replica. + Once the replica receives a heartbeat with a non-zero IPROTO_VCLOCK_SYNC value, + it starts responding with the same value in all its acknowledgements. + This key was introduced in version 2.11. + +* IPROTO_BALLOT_VCLOCK (0x02) is included in the :ref:`IPROTO_BALLOT ` message. + IPROTO_BALLOT is sent in response to the :ref:`IPROTO_VOTE ` request. This key was introduced in :doc:`/release/2.6.1`. -* IPROTO_BALLOT_GC_VCLOCK (0x03) is also sent in response to :ref:`IPROTO_VOTE `. +* IPROTO_BALLOT_GC_VCLOCK (0x03) is also included in the :ref:`IPROTO_BALLOT ` message. + IPROTO_BALLOT is sent in response to the :ref:`IPROTO_VOTE ` request. It is the vclock of the oldest WAL entry on the instance. Corresponds to :ref:`box.info.gc().vclock `. This key was introduced in :doc:`/release/2.6.1`. @@ -680,6 +713,18 @@ They all have the MP_MAP type: It is present only on the instances in the :ref:`"candidate" state ` (IPROTO_RAFT_STATE == 2). +.. _internals-iproto-keys-ballot: + +IPROTO_BALLOT keys +~~~~~~~~~~~~~~~~~~ + +All IPROTO_BALLOT_* keys are only used in the :ref:`IPROTO_BALLOT ` requests. +There have been the following name changes starting with versions :doc:`/release/2.7.3`, :doc:`/release/2.8.2`, +and :doc:`/release/2.10.0`: + +* IPROTO_BALLOT_IS_RO_CFG (0x01) was formerly called IPROTO_BALLOT_IS_RO. +* IPROTO_BALLOT_IS_RO (0x04) was formerly called IPROTO_BALLOT_IS_LOADING. + .. _internals-iproto-keys-metadata: IPROTO_METADATA diff --git a/doc/dev_guide/internals/iproto/replication.rst b/doc/dev_guide/internals/iproto/replication.rst index a607a34a58..9c7a0d7dc1 100644 --- a/doc/dev_guide/internals/iproto/replication.rst +++ b/doc/dev_guide/internals/iproto/replication.rst @@ -64,12 +64,21 @@ Once in :ref:`replication_timeout ` seconds a master sends a :ref:`heartbeat ` message to a replica, and the replica sends a response. Both messages' IPROTO_REQUEST_TYPE is IPROTO_OK. -Note that the master's heartbeat has no body: +IPROTO_TIMESTAMP is a float-64 MP_DOUBLE 8-byte timestamp. + +Since version 2.11, both messages have an optional field in the body that contains +the :ref:`IPROTO_VCLOCK_SYNC ` key. +The master's heartbeat has no body if the IPROTO_VCLOCK_SYNC key is omitted. + +The message from master to a replica: .. raw:: html - :file: images/repl_heartbeat.svg + :file: images/repl_heartbeat_message.svg -IPROTO_TIMESTAMP is a float-64 MP_DOUBLE 8-byte timestamp. +The response from the replica: + +.. raw:: html + :file: images/repl_heartbeat_response.svg The tutorial :ref:`Understanding the binary protocol ` shows actual byte codes of the above heartbeat examples. @@ -122,7 +131,7 @@ its instance ID and its LSN, independent from other masters. IPROTO_ID_FILTER (0x51) is an optional key used in the SUBSCRIBE request followed by an array of ids of instances whose rows won't be relayed to the replica. -The field is encoded only when the id list is not empty. +The field is encoded only when the ID list is not empty. .. _internals-iproto-replication-vote: @@ -146,8 +155,8 @@ IPROTO_BALLOT Code: 0x29. -This value of IPROTO_REQUEST_TYPE indicates a message sent in response to IPROTO_VOTE -(not to be confused with the key IPROTO_RAFT_VOTE). +This value of IPROTO_REQUEST_TYPE indicates a message sent in response +to :ref:`IPROTO_VOTE ` (not to be confused with the key IPROTO_RAFT_VOTE). IPROTO_BALLOT and IPROTO_VOTE are critical during replica set bootstrap. IPROTO_BALLOT corresponds to a map containing the following fields: @@ -155,6 +164,9 @@ IPROTO_BALLOT corresponds to a map containing the following fields: .. raw:: html :file: images/repl_ballot.svg +IPROTO_BALLOT_REGISTERED_REPLICA_UUIDS has the MP_ARRAY type. +The array contains MP_STR elements. + .. _internals-iproto-replication-synchronous: Synchronous @@ -168,11 +180,11 @@ Synchronous * - Name - Code - - Description + - Description * - :ref:`IPROTO_RAFT ` - 0x1e - - Inform that the node changed its RAFT status + - Inform that the node changed its :ref:`RAFT status ` * - :ref:`IPROTO_RAFT_PROMOTE ` - 0x1f @@ -223,7 +235,6 @@ IPROTO_RAFT_PROMOTE Code: 0x1f. See :ref:`box.ctl.promote() `. -Here is what the request contains: .. raw:: html :file: images/repl_raft_promote.svg @@ -248,7 +259,6 @@ IPROTO_RAFT_DEMOTE Code: 0x20. See :ref:`box.ctl.demote() `. -Here is what the request contains: .. raw:: html :file: images/repl_raft_demote.svg diff --git a/doc/how-to/other/iproto.rst b/doc/how-to/other/iproto.rst index 9ea88413ef..fd0f2dada5 100644 --- a/doc/how-to/other/iproto.rst +++ b/doc/how-to/other/iproto.rst @@ -304,16 +304,23 @@ Byte code for the :ref:`heartbeat ` example. The master 04 Main-Map Item #3 IPROTO_TIMESTAMP cb MP_DOUBLE (MessagePack "Float 64") 41 d7 ba 06 7b 3a 03 21 8-byte timestamp + 81 MP_MAP (body), size 1 + 5a Body Map Item #1 IPROTO_VCLOCK_SYNC + 14 MP_UINT = 20 (vclock sync value) -Byte code for the :ref:`heartbeat ` example. The replica might send back this body +Byte code for the :ref:`heartbeat ` example. The replica might send back this body: .. code-block:: none 81 MP_MAP, size 1 00 Main-Map Item #1 IPROTO_REQUEST_TYPE 00 MP_UINT = 0 = IPROTO_OK - 81 Main-Map Item #2, MP_MAP, size 1 - 26 Sub-Map Item #1 IPROTO_VCLOCK - 81 Sub-Map Item #2, MP_MAP, size 1 - 01 MP_UINT = 1 = id (part 1 of vclock) - 06 MP_UINT = 6 = lsn (part 2 of vclock) + 83 MP_MAP (body), size 3 + 26 Body Map Item #1 IPROTO_VCLOCK + 81 MP_MAP, size 1 (vclock of 1 component) + 01 MP_UINT = 1 = id (part 1 of vclock) + 06 MP_UINT = 6 = lsn (part 2 of vclock) + 5a Body Map Item #2 IPROTO_VCLOCK_SYNC + 14 MP_UINT = 20 (vclock sync value) + 53 Body Map Item #3 IPROTO_TERM + 31 MP_UINT = 49 (term value) \ No newline at end of file