Skip to content

Commit 0be1243

Browse files
EvgenyMekhanikKorablev77
EvgenyMekhanik
authored and
Korablev77
committed
Change the behavior of the option 'force_recovery'
There was an option 'force_recovery' that makes tarantool to ignore some problems during xlog recovery. This patch change this option behavior and makes tarantool to ignore some errors during snapshot recovery just like during xlog recovery. Error types which can be ignored: - snapshot is someway truncated, but after necessary system spaces - snapshot has some garbage after it declared length - single tuple within snapshot has broken checksum and may be skipped without consequences (in this case we ignore all row with this tuple) @TarantoolBot document Title: Change 'force_recovery' option behavior Change 'force_recovery' option behavior to allow tarantool loading from broken snapshot Closes #5422
1 parent 141802b commit 0be1243

File tree

4 files changed

+453
-11
lines changed

4 files changed

+453
-11
lines changed

‎src/box/memtx_engine.c

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ memtx_engine_shutdown(struct engine *engine)
150150

151151
static int
152152
memtx_engine_recover_snapshot_row(struct memtx_engine *memtx,
153-
struct xrow_header *row);
153+
struct xrow_header *row, int *is_space_system);
154154

155155
int
156156
memtx_engine_recover_snapshot(struct memtx_engine *memtx,
@@ -170,12 +170,19 @@ memtx_engine_recover_snapshot(struct memtx_engine *memtx,
170170
int rc;
171171
struct xrow_header row;
172172
uint64_t row_count = 0;
173-
while ((rc = xlog_cursor_next(&cursor, &row,
174-
memtx->force_recovery)) == 0) {
173+
int is_space_system = -1;
174+
bool force_recovery = false;
175+
/*
176+
* In case when we read system space, we can't ignore errors.
177+
*/
178+
while ((rc = xlog_cursor_next(&cursor, &row, force_recovery)) == 0) {
175179
row.lsn = signature;
176-
rc = memtx_engine_recover_snapshot_row(memtx, &row);
180+
rc = memtx_engine_recover_snapshot_row(memtx, &row,
181+
&is_space_system);
182+
force_recovery = is_space_system == 0 ?
183+
memtx->force_recovery : false;
177184
if (rc < 0) {
178-
if (!memtx->force_recovery)
185+
if (!force_recovery)
179186
break;
180187
say_error("can't apply row: ");
181188
diag_log();
@@ -188,16 +195,20 @@ memtx_engine_recover_snapshot(struct memtx_engine *memtx,
188195
}
189196
}
190197
xlog_cursor_close(&cursor, false);
191-
if (rc < 0)
198+
if (rc < 0 || is_space_system < 0)
192199
return -1;
193200

194201
/**
195202
* We should never try to read snapshots with no EOF
196203
* marker - such snapshots are very likely corrupted and
197204
* should not be trusted.
198205
*/
199-
if (!xlog_cursor_is_eof(&cursor))
200-
panic("snapshot `%s' has no EOF marker", filename);
206+
if (!xlog_cursor_is_eof(&cursor)) {
207+
if (!memtx->force_recovery)
208+
panic("snapshot `%s' has no EOF marker", filename);
209+
else
210+
say_error("snapshot `%s' has no EOF marker", filename);
211+
}
201212

202213
return 0;
203214
}
@@ -216,7 +227,7 @@ memtx_engine_recover_raft(const struct xrow_header *row)
216227

217228
static int
218229
memtx_engine_recover_snapshot_row(struct memtx_engine *memtx,
219-
struct xrow_header *row)
230+
struct xrow_header *row, int *is_space_system)
220231
{
221232
assert(row->bodycnt == 1); /* always 1 for read */
222233
if (row->type != IPROTO_INSERT) {
@@ -230,6 +241,7 @@ memtx_engine_recover_snapshot_row(struct memtx_engine *memtx,
230241
struct request request;
231242
if (xrow_decode_dml(row, &request, dml_request_key_map(row->type)) != 0)
232243
return -1;
244+
*is_space_system = (request.space_id < BOX_SYSTEM_ID_MAX);
233245
struct space *space = space_cache_find(request.space_id);
234246
if (space == NULL)
235247
return -1;
@@ -448,10 +460,10 @@ memtx_engine_bootstrap(struct engine *engine)
448460
sizeof(bootstrap_bin), "bootstrap") < 0)
449461
return -1;
450462

451-
int rc;
463+
int rc, is_space_system;
452464
struct xrow_header row;
453465
while ((rc = xlog_cursor_next(&cursor, &row, true)) == 0) {
454-
rc = memtx_engine_recover_snapshot_row(memtx, &row);
466+
rc = memtx_engine_recover_snapshot_row(memtx, &row, &is_space_system);
455467
if (rc < 0)
456468
break;
457469
}

‎test/box/gh-5422-broken_snapshot.lua

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/env tarantool
2+
3+
require('console').listen(os.getenv('ADMIN'))
4+
5+
box.cfg({
6+
listen = os.getenv("LISTEN"),
7+
force_recovery = true,
8+
read_only = false,
9+
})

‎test/box/gh-5422-broken_snapshot.result

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,299 @@
1+
-- write data recover from latest snapshot
2+
env = require('test_run')
3+
---
4+
...
5+
test_run = env.new()
6+
---
7+
...
8+
test_run:cmd("restart server default")
9+
fio = require 'fio'
10+
---
11+
...
12+
test_run:cmd("setopt delimiter ';'")
13+
---
14+
- true
15+
...
16+
function get_snap_file ()
17+
local snapfile = nil
18+
local directory = fio.pathjoin(fio.cwd(), 'gh-5422-broken_snapshot')
19+
for files in io.popen(string.format("ls %s", directory)):lines() do
20+
local snaps = string.find(files, "snap")
21+
if (snaps ~= nil) then
22+
local snap = string.find(files, "%n")
23+
if (snap ~= nil) then
24+
snapfile = string.format("%s/%s", directory, files)
25+
end
26+
end
27+
end
28+
return snapfile
29+
end;
30+
---
31+
...
32+
test_run:cmd("setopt delimiter ''");
33+
---
34+
- true
35+
...
36+
test_run:cmd("create server test with script='box/gh-5422-broken_snapshot.lua'")
37+
---
38+
- true
39+
...
40+
test_run:cmd("start server test")
41+
---
42+
- true
43+
...
44+
test_run:cmd("switch test")
45+
---
46+
- true
47+
...
48+
space = box.schema.space.create('test', { engine = "memtx" })
49+
---
50+
...
51+
space:format({ {name = 'id', type = 'unsigned'}, {name = 'year', type = 'unsigned'} })
52+
---
53+
...
54+
index = space:create_index('primary', { parts = {'id'} })
55+
---
56+
...
57+
for key = 1, 10000 do space:insert({key, key + 1000}) end
58+
---
59+
...
60+
box.snapshot()
61+
---
62+
- ok
63+
...
64+
test_run:cmd("switch default")
65+
---
66+
- true
67+
...
68+
snapfile = get_snap_file()
69+
---
70+
...
71+
file = io.open(snapfile, "r")
72+
---
73+
...
74+
size = file:seek("end")
75+
---
76+
...
77+
if size > 30000 then size = 30000 end
78+
---
79+
...
80+
io.close(file)
81+
---
82+
- true
83+
...
84+
-- save last snapshot
85+
os.execute(string.format('cp %s %s.save', snapfile, snapfile))
86+
---
87+
- 0
88+
...
89+
-- write garbage at the end of file
90+
file = io.open(snapfile, "ab")
91+
---
92+
...
93+
for i = 1, 1000, 1 do file:write(math.random(1,254)) end
94+
---
95+
...
96+
io.close(file)
97+
---
98+
- true
99+
...
100+
test_run:cmd("switch test")
101+
---
102+
- true
103+
...
104+
test_run:cmd("restart server test with script='box/gh-5422-broken_snapshot.lua'")
105+
test_run:cmd("setopt delimiter ';'")
106+
---
107+
- true
108+
...
109+
-- check that all data valid
110+
val = box.space.test:select()
111+
for i = 1, 10000, 1 do
112+
assert(val[i] ~= nil)
113+
end;
114+
---
115+
...
116+
test_run:cmd("setopt delimiter ''");
117+
---
118+
- true
119+
...
120+
test_run:cmd("switch default")
121+
---
122+
- true
123+
...
124+
-- restore snapshot
125+
os.execute(string.format('cp %s.save %s', snapfile, snapfile))
126+
---
127+
- 0
128+
...
129+
-- truncate
130+
os.execute(string.format('dd if=%s.save of=%s bs=%d count=1', snapfile, snapfile, size))
131+
---
132+
- 0
133+
...
134+
test_run:cmd("switch test")
135+
---
136+
- true
137+
...
138+
test_run:cmd("restart server test with script='box/gh-5422-broken_snapshot.lua'")
139+
-- check than some data valid
140+
test_run:cmd("setopt delimiter ';'")
141+
---
142+
- true
143+
...
144+
val = box.space.test:select();
145+
---
146+
...
147+
for i = 1, 1000, 1 do
148+
assert(val[i] ~= nil)
149+
end;
150+
---
151+
...
152+
test_run:cmd("setopt delimiter ''");
153+
---
154+
- true
155+
...
156+
test_run:cmd("switch default")
157+
---
158+
- true
159+
...
160+
-- restore snapshot
161+
os.execute(string.format('cp %s.save %s', snapfile, snapfile))
162+
---
163+
- 0
164+
...
165+
-- write garbage at the middle of file
166+
file = io.open(snapfile, "r+b")
167+
---
168+
...
169+
file:seek("set", size)
170+
---
171+
- 30000
172+
...
173+
for i = 1, 100, 1 do file:write(math.random(1,254)) end
174+
---
175+
...
176+
io.close(file)
177+
---
178+
- true
179+
...
180+
test_run:cmd("switch test")
181+
---
182+
- true
183+
...
184+
test_run:cmd("restart server test with script='box/gh-5422-broken_snapshot.lua'")
185+
test_run:cmd("setopt delimiter ';'")
186+
---
187+
- true
188+
...
189+
-- check that some data valid
190+
val = box.space.test:select();
191+
---
192+
...
193+
for i = 1, 1000, 1 do
194+
assert(val[i] ~= nil)
195+
end;
196+
---
197+
...
198+
test_run:cmd("setopt delimiter ''");
199+
---
200+
- true
201+
...
202+
test_run:cmd("switch default")
203+
---
204+
- true
205+
...
206+
-- restore snapshot
207+
os.execute(string.format('cp %s.save %s', snapfile, snapfile))
208+
---
209+
- 0
210+
...
211+
-- write big garbage at the middle of file, check that start data valid
212+
file = io.open(snapfile, "r+b")
213+
---
214+
...
215+
file:seek("set", size / 2 + 8000)
216+
---
217+
- 23000
218+
...
219+
for i = 1, 10000, 1 do file:write(math.random(1,254)) end
220+
---
221+
...
222+
io.close(file)
223+
---
224+
- true
225+
...
226+
test_run:cmd("switch test")
227+
---
228+
- true
229+
...
230+
test_run:cmd("restart server test with script='box/gh-5422-broken_snapshot.lua'")
231+
test_run:cmd("setopt delimiter ';'")
232+
---
233+
- true
234+
...
235+
-- check that some data valid
236+
val = box.space.test:select();
237+
---
238+
...
239+
for i = 1, 1000, 1 do
240+
assert(val[i] ~= nil)
241+
end;
242+
---
243+
...
244+
test_run:cmd("setopt delimiter ''");
245+
---
246+
- true
247+
...
248+
test_run:cmd("switch default")
249+
---
250+
- true
251+
...
252+
test_run:cmd('stop server test')
253+
---
254+
- true
255+
...
256+
-- restore snapshot
257+
os.execute(string.format('cp %s.save %s', snapfile, snapfile))
258+
---
259+
- 0
260+
...
261+
os.execute(string.format('rm %s.save', snapfile))
262+
---
263+
- 0
264+
...
265+
-- write big garbage at the start of file
266+
file = io.open(snapfile, "r+b")
267+
---
268+
...
269+
file:seek("set", size / 2)
270+
---
271+
- 15000
272+
...
273+
for i = 1, 1000, 1 do file:write(math.random(1,254)) end
274+
---
275+
...
276+
io.close(file)
277+
---
278+
- true
279+
...
280+
test_run:cmd("start server test with crash_expected=True")
281+
---
282+
- false
283+
...
284+
log = string.format("%s/%s.%s", fio.cwd(), "gh-5422-broken_snapshot", "log")
285+
---
286+
...
287+
-- We must not find ER_UNKNOWN_REPLICA in log file, so grep return not 0
288+
assert(os.execute(string.format("cat %s | grep ER_UNKNOWN_REPLICA:", log)) ~= 0)
289+
---
290+
- true
291+
...
292+
test_run:cmd("cleanup server test")
293+
---
294+
- true
295+
...
296+
test_run:cmd("delete server test")
297+
---
298+
- true
299+
...

0 commit comments

Comments
 (0)