basebackup.c Source File (original) (raw)

15#include <sys/stat.h>

26#include "catalog/pg_tablespace_d.h"

60#define SINK_BUFFER_LENGTH Max(32768, BLCKSZ)

62typedef struct

63{

84static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,

85 List *tablespaces, bool sendtblspclinks,

88static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,

89 struct stat *statbuf, bool missing_ok,

91 unsigned segno,

93 unsigned num_incremental_blocks,

95 unsigned truncation_block_length);

97 const char *readfilename, int fd,

98 off_t offset, size_t length,

100 bool verify_checksum,

101 int *checksum_failures);

103 size_t *bytes_done, void *data, size_t length);

106 uint16 *expected_checksum);

108 const char *content, int len,

111 const char *linktarget, struct stat *statbuf,

112 bool sizeonly);

120 const char *filename, bool partial_read_ok);

121

122

124

125

127

128

130

131

132

133

134

135

136

138{

141};

142

143

144

145

146

147

148

149

150

152{

153

154

155

156

158

159

160

161

162

163

165

166

168

169

170 "pg_notify",

171

172

173

174

175

176 "pg_serial",

177

178

179 "pg_snapshots",

180

181

182 "pg_subtrans",

183

184

185 NULL

186};

187

188

189

190

192{

193

195

196

198

199

200

201

202

204

205

206

207

208

211

212

213

214

215

216

217

218 {"backup_manifest", false},

219

220 {"postmaster.pid", false},

221 {"postmaster.opts", false},

222

223

224 {NULL, false}

225};

226

227

228

229

230

231

232

233static void

236{

243

244

246 state.tablespace_num = 0;

247 state.bytes_done = 0;

248 state.bytes_total = 0;

249 state.bytes_total_is_valid = false;

250

251

256

258

261

263

264

267

271

274

275

276

277

278

279

280

281

283 {

286

287

288 if (ib != NULL)

290

291

293 newti->size = -1;

295

296

297

298

299

301 {

303

304 foreach(lc, state.tablespaces)

305 {

307

308 if (tmp->path == NULL)

311 else

313 NULL, NULL);

315 }

316 state.bytes_total_is_valid = true;

317 }

318

319

321

322

323 foreach(lc, state.tablespaces)

324 {

326

327 if (ti->path == NULL)

328 {

329 struct stat statbuf;

330 bool sendtblspclinks = true;

331 char *backup_label;

332

334

335

338 backup_label, -1, &manifest);

339 pfree(backup_label);

340

341

343 {

346 sendtblspclinks = false;

347 }

348

349

350 sendDir(sink, ".", 1, false, state.tablespaces,

352

353

357 errmsg("could not stat file \"%s\": %m",

362 }

363 else

364 {

365 char *archive_name = psprintf("%u.tar", ti->oid);

366

368

370 }

371

372

373

374

375

376

377

379 {

381 }

382 else

383 {

384

386 "BLCKSZ too small for 2 tar blocks");

389

390

392 }

393 }

394

397

400

401

404 }

406

407

409 {

410

411

412

413

418 struct stat statbuf;

419 List *historyFileList = NIL;

423 DIR *dir;

427

429

430

431

432

433

434

435

436

437

438

443

445 while ((de = ReadDir(dir, "pg_wal")) != NULL)

446 {

447

449 strcmp(de->d_name + 8, firstoff + 8) >= 0 &&

450 strcmp(de->d_name + 8, lastoff + 8) <= 0)

451 {

453 }

454

456 {

458 }

459 }

461

462

463

464

465

467

468

469

470

471

472

474

475

476

477

478

479 if (walFileList == NIL)

481 (errmsg("could not find any WAL files")));

482

483

484

485

486

489 if (segno != startsegno)

490 {

492

496 (errmsg("could not find WAL file \"%s\"", startfname)));

497 }

498 foreach(lc, walFileList)

499 {

500 char *walFileName = (char *) lfirst(lc);

503

505 if (!(nextsegno == segno || currsegno == segno))

506 {

508

511 (errmsg("could not find WAL file \"%s\"", nextfname)));

512 }

513 }

514 if (segno != endsegno)

515 {

517

520 (errmsg("could not find WAL file \"%s\"", endfname)));

521 }

522

523

524 foreach(lc, walFileList)

525 {

526 char *walFileName = (char *) lfirst(lc);

527 int fd;

528 ssize_t cnt;

530

533

535 if (fd < 0)

536 {

537 int save_errno = errno;

538

539

540

541

542

543

545

546 errno = save_errno;

549 errmsg("could not open file \"%s\": %m", pathbuf)));

550 }

551

552 if (fstat(fd, &statbuf) != 0)

555 errmsg("could not stat file \"%s\": %m",

556 pathbuf)));

558 {

562 errmsg("unexpected WAL file size \"%s\"", walFileName)));

563 }

564

565

567

571 len, pathbuf, true)) > 0)

572 {

575

576 len += cnt;

577

579 break;

580 }

581

583 {

587 errmsg("unexpected WAL file size \"%s\"", walFileName)));

588 }

589

590

591

592

593

595

597

598

599

600

601

602

603

606 }

607

608

609

610

611

612

613

614

615

616

617 foreach(lc, historyFileList)

618 {

619 char *fname = lfirst(lc);

620

622

623 if (lstat(pathbuf, &statbuf) != 0)

626 errmsg("could not stat file \"%s\": %m", pathbuf)));

627

628 sendFile(sink, pathbuf, pathbuf, &statbuf, false,

631

632

635 }

636

637

639 "BLCKSZ too small for 2 tar blocks");

642

643

645 }

646

648 endptr, endtli);

649

651

653

655 {

658 (errmsg_plural("%lld total checksum verification failure",

659 "%lld total checksum verification failures",

662

665 errmsg("checksum verification failure during base backup")));

666 }

667

668

669

670

671

672

674

675

677

679}

680

681

682

683

684

685static int

687{

688 char *fna = (char *) lfirst(a);

689 char *fnb = (char *) lfirst(b);

690

691 return strcmp(fna + 8, fnb + 8);

692}

693

694

695

696

697static void

699{

701 bool o_label = false;

702 bool o_progress = false;

703 bool o_checkpoint = false;

704 bool o_nowait = false;

705 bool o_wal = false;

706 bool o_incremental = false;

707 bool o_maxrate = false;

708 bool o_tablespace_map = false;

709 bool o_noverify_checksums = false;

710 bool o_manifest = false;

711 bool o_manifest_checksums = false;

712 bool o_target = false;

713 bool o_target_detail = false;

714 char *target_str = NULL;

715 char *target_detail_str = NULL;

716 bool o_compression = false;

717 bool o_compression_detail = false;

718 char *compression_detail_str = NULL;

719

720 MemSet(opt, 0, sizeof(*opt));

725

727 {

729

730 if (strcmp(defel->defname, "label") == 0)

731 {

732 if (o_label)

734 (errcode(ERRCODE_SYNTAX_ERROR),

735 errmsg("duplicate option \"%s\"", defel->defname)));

737 o_label = true;

738 }

739 else if (strcmp(defel->defname, "progress") == 0)

740 {

741 if (o_progress)

743 (errcode(ERRCODE_SYNTAX_ERROR),

744 errmsg("duplicate option \"%s\"", defel->defname)));

746 o_progress = true;

747 }

748 else if (strcmp(defel->defname, "checkpoint") == 0)

749 {

751

752 if (o_checkpoint)

754 (errcode(ERRCODE_SYNTAX_ERROR),

755 errmsg("duplicate option \"%s\"", defel->defname)));

760 else

762 (errcode(ERRCODE_SYNTAX_ERROR),

763 errmsg("unrecognized checkpoint type: \"%s\"",

764 optval)));

765 o_checkpoint = true;

766 }

767 else if (strcmp(defel->defname, "wait") == 0)

768 {

769 if (o_nowait)

771 (errcode(ERRCODE_SYNTAX_ERROR),

772 errmsg("duplicate option \"%s\"", defel->defname)));

774 o_nowait = true;

775 }

776 else if (strcmp(defel->defname, "wal") == 0)

777 {

778 if (o_wal)

780 (errcode(ERRCODE_SYNTAX_ERROR),

781 errmsg("duplicate option \"%s\"", defel->defname)));

783 o_wal = true;

784 }

785 else if (strcmp(defel->defname, "incremental") == 0)

786 {

787 if (o_incremental)

789 (errcode(ERRCODE_SYNTAX_ERROR),

790 errmsg("duplicate option \"%s\"", defel->defname)));

794 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),

795 errmsg("incremental backups cannot be taken unless WAL summarization is enabled")));

796 o_incremental = true;

797 }

798 else if (strcmp(defel->defname, "max_rate") == 0)

799 {

801

802 if (o_maxrate)

804 (errcode(ERRCODE_SYNTAX_ERROR),

805 errmsg("duplicate option \"%s\"", defel->defname)));

806

808 if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)

810 (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),

811 errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",

813

815 o_maxrate = true;

816 }

817 else if (strcmp(defel->defname, "tablespace_map") == 0)

818 {

819 if (o_tablespace_map)

821 (errcode(ERRCODE_SYNTAX_ERROR),

822 errmsg("duplicate option \"%s\"", defel->defname)));

824 o_tablespace_map = true;

825 }

826 else if (strcmp(defel->defname, "verify_checksums") == 0)

827 {

828 if (o_noverify_checksums)

830 (errcode(ERRCODE_SYNTAX_ERROR),

831 errmsg("duplicate option \"%s\"", defel->defname)));

833 o_noverify_checksums = true;

834 }

835 else if (strcmp(defel->defname, "manifest") == 0)

836 {

838 bool manifest_bool;

839

840 if (o_manifest)

842 (errcode(ERRCODE_SYNTAX_ERROR),

843 errmsg("duplicate option \"%s\"", defel->defname)));

844 if (parse_bool(optval, &manifest_bool))

845 {

846 if (manifest_bool)

848 else

850 }

851 else if (pg_strcasecmp(optval, "force-encode") == 0)

853 else

855 (errcode(ERRCODE_SYNTAX_ERROR),

856 errmsg("unrecognized manifest option: \"%s\"",

857 optval)));

858 o_manifest = true;

859 }

860 else if (strcmp(defel->defname, "manifest_checksums") == 0)

861 {

863

864 if (o_manifest_checksums)

866 (errcode(ERRCODE_SYNTAX_ERROR),

867 errmsg("duplicate option \"%s\"", defel->defname)));

871 (errcode(ERRCODE_SYNTAX_ERROR),

872 errmsg("unrecognized checksum algorithm: \"%s\"",

873 optval)));

874 o_manifest_checksums = true;

875 }

876 else if (strcmp(defel->defname, "target") == 0)

877 {

878 if (o_target)

880 (errcode(ERRCODE_SYNTAX_ERROR),

881 errmsg("duplicate option \"%s\"", defel->defname)));

883 o_target = true;

884 }

885 else if (strcmp(defel->defname, "target_detail") == 0)

886 {

888

889 if (o_target_detail)

891 (errcode(ERRCODE_SYNTAX_ERROR),

892 errmsg("duplicate option \"%s\"", defel->defname)));

893 target_detail_str = optval;

894 o_target_detail = true;

895 }

896 else if (strcmp(defel->defname, "compression") == 0)

897 {

899

900 if (o_compression)

902 (errcode(ERRCODE_SYNTAX_ERROR),

903 errmsg("duplicate option \"%s\"", defel->defname)));

906 (errcode(ERRCODE_SYNTAX_ERROR),

907 errmsg("unrecognized compression algorithm: \"%s\"",

908 optval)));

909 o_compression = true;

910 }

911 else if (strcmp(defel->defname, "compression_detail") == 0)

912 {

913 if (o_compression_detail)

915 (errcode(ERRCODE_SYNTAX_ERROR),

916 errmsg("duplicate option \"%s\"", defel->defname)));

917 compression_detail_str = defGetString(defel);

918 o_compression_detail = true;

919 }

920 else

922 (errcode(ERRCODE_SYNTAX_ERROR),

923 errmsg("unrecognized base backup option: \"%s\"",

925 }

926

927 if (opt->label == NULL)

928 opt->label = "base backup";

930 {

931 if (o_manifest_checksums)

933 (errcode(ERRCODE_SYNTAX_ERROR),

934 errmsg("manifest checksums require a backup manifest")));

936 }

937

938 if (target_str == NULL)

939 {

940 if (target_detail_str != NULL)

942 (errcode(ERRCODE_SYNTAX_ERROR),

943 errmsg("target detail cannot be used without target")));

946 }

947 else if (strcmp(target_str, "client") == 0)

948 {

949 if (target_detail_str != NULL)

951 (errcode(ERRCODE_SYNTAX_ERROR),

952 errmsg("target \"%s\" does not accept a target detail",

953 target_str)));

955 }

956 else

959

960 if (o_compression_detail && !o_compression)

962 (errcode(ERRCODE_SYNTAX_ERROR),

963 errmsg("compression detail cannot be specified unless compression is enabled")));

964

965 if (o_compression)

966 {

967 char *error_detail;

968

971 error_detail =

973 if (error_detail != NULL)

975 errcode(ERRCODE_SYNTAX_ERROR),

976 errmsg("invalid compression specification: %s",

977 error_detail));

978 }

979}

980

981

982

983

984

985

986

987

988

989void

991{

995

998 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),

999 errmsg("a backup is already in progress in this session")));

1000

1002

1004

1006 {

1007 char activitymsg[50];

1008

1009 snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",

1012 }

1013

1014

1015

1016

1017

1018

1019

1020

1022 ib = NULL;

1023 else if (ib == NULL)

1025 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),

1026 errmsg("must UPLOAD_MANIFEST before performing an incremental BASE_BACKUP")));

1027

1028

1029

1030

1031

1032

1033

1037

1038

1041

1042

1049

1050

1052

1053

1054

1055

1056

1058 {

1060 }

1062 {

1064 }

1066}

1067

1068

1069

1070

1071

1072

1073

1074static void

1077{

1078 struct stat statbuf;

1079 int bytes_done = 0;

1081

1083 elog(ERROR, "could not initialize checksum of file \"%s\"",

1085

1086 if (len < 0)

1087 len = strlen(content);

1088

1089

1090

1091

1092

1093

1094#ifdef WIN32

1097#else

1098 statbuf.st_uid = geteuid();

1099 statbuf.st_gid = getegid();

1100#endif

1101 statbuf.st_mtime = time(NULL);

1104

1106

1108 elog(ERROR, "could not update checksum of file \"%s\"",

1110

1111 while (bytes_done < len)

1112 {

1115

1116 memcpy(sink->bbs_buffer, content, nbytes);

1118 bytes_done += nbytes;

1119 content += nbytes;

1120 }

1121

1123

1126}

1127

1128

1129

1130

1131

1132

1133

1134

1138{

1141 struct stat statbuf;

1142

1143

1144

1145

1146

1147 snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,

1149

1150

1151

1152

1153

1154 if (lstat(pathbuf, &statbuf) != 0)

1155 {

1156 if (errno != ENOENT)

1159 errmsg("could not stat file or directory \"%s\": %m",

1160 pathbuf)));

1161

1162

1163 return 0;

1164 }

1165

1167 sizeonly);

1168

1169

1170 size += sendDir(sink, pathbuf, strlen(path), sizeonly, NIL, true, manifest,

1171 spcoid, ib);

1172

1173 return size;

1174}

1175

1176

1177

1178

1179

1180

1181

1182

1183

1184

1185

1186

1187

1189 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,

1192{

1193 DIR *dir;

1196 struct stat statbuf;

1198 const char *lastDir;

1199 bool isRelationDir = false;

1200 bool isGlobalDir = false;

1202 BlockNumber *relative_block_numbers = NULL;

1203

1204

1205

1206

1207

1208 if (ib != NULL)

1209 relative_block_numbers = palloc(sizeof(BlockNumber) * RELSEG_SIZE);

1210

1211

1212

1213

1214

1215

1216

1217

1219

1220

1221 if (lastDir != NULL &&

1222 strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))

1223 {

1224

1225 int parentPathLen = lastDir - path;

1226

1227

1228

1229

1230

1231 if (strncmp(path, "./base", parentPathLen) == 0 ||

1236 {

1237 isRelationDir = true;

1238 dboid = atooid(lastDir + 1);

1239 }

1240 }

1241 else if (strcmp(path, "./global") == 0)

1242 {

1243 isRelationDir = true;

1244 isGlobalDir = true;

1245 }

1246

1248 while ((de = ReadDir(dir, path)) != NULL)

1249 {

1250 int excludeIdx;

1251 bool excludeFound;

1254 unsigned segno = 0;

1255 bool isRelationFile = false;

1256

1257

1258 if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)

1259 continue;

1260

1261

1262 if (strncmp(de->d_name,

1265 continue;

1266

1267

1268 if (strcmp(de->d_name, ".DS_Store") == 0)

1269 continue;

1270

1271

1272

1273

1274

1275

1276

1277

1278

1282 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),

1283 errmsg("the standby was promoted during online backup"),

1284 errhint("This means that the backup being taken is corrupt "

1285 "and should not be used. "

1286 "Try taking another online backup.")));

1287

1288

1289 excludeFound = false;

1290 for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)

1291 {

1293

1295 cmplen++;

1297 {

1298 elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);

1299 excludeFound = true;

1300 break;

1301 }

1302 }

1303

1304 if (excludeFound)

1305 continue;

1306

1307

1308

1309

1310

1311 if (isRelationDir)

1312 isRelationFile =

1314 &relfilenumber,

1315 &relForkNum, &segno);

1316

1317

1318 if (isRelationFile && relForkNum != INIT_FORKNUM)

1319 {

1321

1322

1323

1324

1325

1326 snprintf(initForkFile, sizeof(initForkFile), "%s/%u_init",

1327 path, relfilenumber);

1328

1329 if (lstat(initForkFile, &statbuf) == 0)

1330 {

1332 "unlogged relation file \"%s\" excluded from backup",

1334

1335 continue;

1336 }

1337 }

1338

1339

1341 {

1343 "temporary relation file \"%s\" excluded from backup",

1345

1346 continue;

1347 }

1348

1349 snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);

1350

1351

1353 continue;

1354

1355 if (lstat(pathbuf, &statbuf) != 0)

1356 {

1357 if (errno != ENOENT)

1360 errmsg("could not stat file or directory \"%s\": %m",

1361 pathbuf)));

1362

1363

1364 continue;

1365 }

1366

1367

1368 excludeFound = false;

1369 for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)

1370 {

1372 {

1373 elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);

1375 size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,

1376 &statbuf, sizeonly);

1377 excludeFound = true;

1378 break;

1379 }

1380 }

1381

1382 if (excludeFound)

1383 continue;

1384

1385

1386

1387

1388

1389

1390 if (strcmp(pathbuf, "./pg_wal") == 0)

1391 {

1392

1394 size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,

1395 &statbuf, sizeonly);

1396

1397

1398

1399

1400

1401 size += _tarWriteHeader(sink, "./pg_wal/archive_status", NULL,

1402 &statbuf, sizeonly);

1403 size += _tarWriteHeader(sink, "./pg_wal/summaries", NULL,

1404 &statbuf, sizeonly);

1405

1406 continue;

1407 }

1408

1409

1410 if (strcmp(path, "./pg_tblspc") == 0 && S_ISLNK(statbuf.st_mode))

1411 {

1413 int rllen;

1414

1415 rllen = readlink(pathbuf, linkpath, sizeof(linkpath));

1416 if (rllen < 0)

1419 errmsg("could not read symbolic link \"%s\": %m",

1420 pathbuf)));

1421 if (rllen >= sizeof(linkpath))

1423 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),

1424 errmsg("symbolic link \"%s\" target is too long",

1425 pathbuf)));

1426 linkpath[rllen] = '\0';

1427

1428 size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, linkpath,

1429 &statbuf, sizeonly);

1430 }

1432 {

1433 bool skip_this_dir = false;

1435

1436

1437

1438

1439

1440 size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, &statbuf,

1441 sizeonly);

1442

1443

1444

1445

1446

1447 foreach(lc, tablespaces)

1448 {

1450

1451

1452

1453

1454

1455

1456

1457

1458 if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)

1459 {

1460 skip_this_dir = true;

1461 break;

1462 }

1463 }

1464

1465

1466

1467

1468 if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)

1469 skip_this_dir = true;

1470

1471 if (!skip_this_dir)

1472 size += sendDir(sink, pathbuf, basepathlen, sizeonly, tablespaces,

1473 sendtblspclinks, manifest, spcoid, ib);

1474 }

1476 {

1477 bool sent = false;

1478 unsigned num_blocks_required = 0;

1479 unsigned truncation_block_length = 0;

1480 char tarfilenamebuf[MAXPGPATH * 2];

1481 char *tarfilename = pathbuf + basepathlen + 1;

1483

1484 if (ib != NULL && isRelationFile)

1485 {

1486 Oid relspcoid;

1487 char *lookup_path;

1488

1490 {

1491 relspcoid = spcoid;

1493 tarfilename);

1494 }

1495 else

1496 {

1497 if (isGlobalDir)

1498 relspcoid = GLOBALTABLESPACE_OID;

1499 else

1500 relspcoid = DEFAULTTABLESPACE_OID;

1501 lookup_path = pstrdup(tarfilename);

1502 }

1503

1505 relfilenumber, relForkNum,

1507 &num_blocks_required,

1508 relative_block_numbers,

1509 &truncation_block_length);

1511 {

1514 snprintf(tarfilenamebuf, sizeof(tarfilenamebuf),

1515 "%s/INCREMENTAL.%s",

1516 path + basepathlen + 1,

1518 tarfilename = tarfilenamebuf;

1519 }

1520

1521 pfree(lookup_path);

1522 }

1523

1524 if (!sizeonly)

1525 sent = sendFile(sink, pathbuf, tarfilename, &statbuf,

1526 true, dboid, spcoid,

1527 relfilenumber, segno, manifest,

1528 num_blocks_required,

1530 truncation_block_length);

1531

1532 if (sent || sizeonly)

1533 {

1534

1535 size += statbuf.st_size;

1536

1537

1539

1540

1542 }

1543 }

1544 else

1546 (errmsg("skipping special file \"%s\"", pathbuf)));

1547 }

1548

1549 if (relative_block_numbers != NULL)

1550 pfree(relative_block_numbers);

1551

1553 return size;

1554}

1555

1556

1557

1558

1559

1560

1561

1562

1563

1564

1565

1566

1567

1568

1569

1570

1571

1572

1573static bool

1574 sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,

1575 struct stat *statbuf, bool missing_ok, Oid dboid, Oid spcoid,

1578 BlockNumber *incremental_blocks, unsigned truncation_block_length)

1579{

1580 int fd;

1582 int checksum_failures = 0;

1583 off_t cnt;

1585 bool verify_checksum = false;

1587 int ibindex = 0;

1588

1590 elog(ERROR, "could not initialize checksum of file \"%s\"",

1591 readfilename);

1592

1594 if (fd < 0)

1595 {

1596 if (errno == ENOENT && missing_ok)

1597 return false;

1600 errmsg("could not open file \"%s\": %m", readfilename)));

1601 }

1602

1603 _tarWriteHeader(sink, tarfilename, NULL, statbuf, false);

1604

1605

1606

1607

1608

1610

1611

1612

1613

1614

1615

1618 verify_checksum = true;

1619

1620

1621

1622

1623 if (incremental_blocks != NULL)

1624 {

1626 size_t header_bytes_done = 0;

1627 char padding[BLCKSZ];

1628 size_t paddinglen;

1629

1630

1631 push_to_sink(sink, &checksum_ctx, &header_bytes_done,

1632 &magic, sizeof(magic));

1633 push_to_sink(sink, &checksum_ctx, &header_bytes_done,

1634 &num_incremental_blocks, sizeof(num_incremental_blocks));

1635 push_to_sink(sink, &checksum_ctx, &header_bytes_done,

1636 &truncation_block_length, sizeof(truncation_block_length));

1637 push_to_sink(sink, &checksum_ctx, &header_bytes_done,

1638 incremental_blocks,

1639 sizeof(BlockNumber) * num_incremental_blocks);

1640

1641

1642

1643

1644

1645

1646

1647

1648 if ((num_incremental_blocks > 0) && (header_bytes_done % BLCKSZ != 0))

1649 {

1650 paddinglen = (BLCKSZ - (header_bytes_done % BLCKSZ));

1651

1652 memset(padding, 0, paddinglen);

1653 bytes_done += paddinglen;

1654

1655 push_to_sink(sink, &checksum_ctx, &header_bytes_done,

1656 padding, paddinglen);

1657 }

1658

1659

1660 if (header_bytes_done > 0)

1661 {

1665 header_bytes_done) < 0)

1666 elog(ERROR, "could not update checksum of base backup");

1667 }

1668

1669

1670 bytes_done += sizeof(magic);

1671 bytes_done += sizeof(num_incremental_blocks);

1672 bytes_done += sizeof(truncation_block_length);

1673 bytes_done += sizeof(BlockNumber) * num_incremental_blocks;

1674 }

1675

1676

1677

1678

1679

1680

1681

1682 while (1)

1683 {

1684

1685

1686

1687

1688 if (incremental_blocks == NULL)

1689 {

1691

1692

1693

1694

1695

1696 if (bytes_done >= statbuf->st_size)

1697 break;

1698

1699

1700

1701

1702

1705 blkno + segno * RELSEG_SIZE,

1706 verify_checksum,

1707 &checksum_failures);

1708 }

1709 else

1710 {

1712

1713

1714

1715

1716 if (ibindex >= num_incremental_blocks)

1717 break;

1718

1719

1720

1721

1722

1723 relative_blkno = incremental_blocks[ibindex++];

1725 relative_blkno * BLCKSZ,

1726 BLCKSZ,

1727 relative_blkno + segno * RELSEG_SIZE,

1728 verify_checksum,

1729 &checksum_failures);

1730

1731

1732

1733

1734

1735

1736

1737

1738

1739

1740

1741

1742 if (cnt < BLCKSZ)

1743 break;

1744 }

1745

1746

1747

1748

1749

1750 if (verify_checksum && (cnt % BLCKSZ != 0))

1751 {

1753 (errmsg("could not verify checksum in file \"%s\", block "

1754 "%u: read buffer size %d and page size %d "

1755 "differ",

1756 readfilename, blkno, (int) cnt, BLCKSZ)));

1757 verify_checksum = false;

1758 }

1759

1760

1761

1762

1763

1764

1765 if (cnt == 0)

1766 break;

1767

1768

1769 blkno += cnt / BLCKSZ;

1770 bytes_done += cnt;

1771

1772

1773

1774

1775

1776 Assert(!((incremental_blocks != NULL && num_incremental_blocks > 0) &&

1777 (bytes_done % BLCKSZ != 0)));

1778

1779

1781

1782

1785 elog(ERROR, "could not update checksum of base backup");

1786 }

1787

1788

1789 while (bytes_done < statbuf->st_size)

1790 {

1793

1797 nbytes) < 0)

1798 elog(ERROR, "could not update checksum of base backup");

1800 bytes_done += nbytes;

1801 }

1802

1803

1804

1805

1806

1807

1809

1811

1812 if (checksum_failures > 1)

1813 {

1815 (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",

1816 "file \"%s\" has a total of %d checksum verification failures",

1817 checksum_failures,

1818 readfilename, checksum_failures)));

1819

1822 }

1823

1825

1828

1829 return true;

1830}

1831

1832

1833

1834

1835

1836

1837

1838

1839

1840

1841

1842

1843

1844

1845

1846

1847

1848

1849static off_t

1851 off_t offset, size_t length, BlockNumber blkno,

1852 bool verify_checksum, int *checksum_failures)

1853{

1854 off_t cnt;

1855 int i;

1856 char *page;

1857

1858

1861 offset, readfilename, true);

1862

1863

1864 if (!verify_checksum || (cnt % BLCKSZ) != 0)

1865 return cnt;

1866

1867

1868 for (i = 0; i < cnt / BLCKSZ; i++)

1869 {

1870 int reread_cnt;

1871 uint16 expected_checksum;

1872

1874

1875

1877 &expected_checksum))

1878 continue;

1879

1880

1881

1882

1883

1884

1885

1886

1887

1888

1889

1890

1891

1892

1893

1894

1895

1896 reread_cnt =

1898 BLCKSZ, offset + BLCKSZ * i,

1899 readfilename, false);

1900 if (reread_cnt == 0)

1901 {

1902

1903

1904

1905

1906

1907 cnt = BLCKSZ * i;

1908 break;

1909 }

1910

1911

1913 &expected_checksum))

1914 continue;

1915

1916

1917 (*checksum_failures)++;

1918 if (*checksum_failures <= 5)

1920 (errmsg("checksum verification failed in "

1921 "file \"%s\", block %u: calculated "

1922 "%X but expected %X",

1923 readfilename, blkno + i, expected_checksum,

1924 ((PageHeader) page)->pd_checksum)));

1925 if (*checksum_failures == 5)

1927 (errmsg("further checksum verification "

1928 "failures in file \"%s\" will not "

1929 "be reported", readfilename)));

1930 }

1931

1932 return cnt;

1933}

1934

1935

1936

1937

1938

1939

1940

1941

1942

1943

1944

1945

1946

1947

1948

1949

1950

1951

1952static void

1954 size_t *bytes_done, void *data, size_t length)

1955{

1956 while (length > 0)

1957 {

1958 size_t bytes_to_copy;

1959

1960

1961

1962

1963

1964 if (length < sink->bbs_buffer_length - *bytes_done)

1965 {

1966

1967 memcpy(sink->bbs_buffer + *bytes_done, data, length);

1968 *bytes_done += length;

1969 return;

1970 }

1971

1972

1974 memcpy(sink->bbs_buffer + *bytes_done, data, bytes_to_copy);

1975 data = ((char *) data) + bytes_to_copy;

1976 length -= bytes_to_copy;

1980 elog(ERROR, "could not update checksum");

1981 *bytes_done = 0;

1982 }

1983}

1984

1985

1986

1987

1988

1989

1990

1991

1992

1993static bool

1995 uint16 *expected_checksum)

1996{

1999

2000

2001

2002

2003

2004

2005

2006

2008 return true;

2009

2010

2012

2013

2016 return true;

2017 *expected_checksum = checksum;

2018 return false;

2019}

2020

2023 struct stat *statbuf, bool sizeonly)

2024{

2026

2027 if (!sizeonly)

2028 {

2029

2030

2031

2032

2033

2034

2035

2037 "BLCKSZ too small for tar block");

2039

2044

2045 switch (rc)

2046 {

2048 break;

2051 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),

2052 errmsg("file name too long for tar format: \"%s\"",

2054 break;

2057 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),

2058 errmsg("symbolic link target too long for tar format: "

2059 "file name \"%s\", target \"%s\"",

2061 break;

2062 default:

2063 elog(ERROR, "unrecognized tar error: %d", rc);

2064 }

2065

2067 }

2068

2070}

2071

2072

2073

2074

2075static void

2077{

2079

2080

2081

2082

2083

2086

2087 if (pad > 0)

2088 {

2091 }

2092}

2093

2094

2095

2096

2097

2098static void

2100{

2101

2104}

2105

2106

2107

2108

2109

2110

2111

2112

2113

2114

2115static ssize_t

2117 const char *filename, bool partial_read_ok)

2118{

2119 ssize_t rc;

2120

2124

2125 if (rc < 0)

2128 errmsg("could not read file \"%s\": %m", filename)));

2129 if (!partial_read_ok && rc > 0 && rc != nbytes)

2132 errmsg("could not read file \"%s\": read %zd of %zu",

2134

2135 return rc;

2136}

void InitializeBackupManifest(backup_manifest_info *manifest, backup_manifest_option want_manifest, pg_checksum_type manifest_checksum_type)

void AddFileToBackupManifest(backup_manifest_info *manifest, Oid spcoid, const char *pathname, size_t size, pg_time_t mtime, pg_checksum_context *checksum_ctx)

void AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr, TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)

void SendBackupManifest(backup_manifest_info *manifest, bbsink *sink)

void FreeBackupManifest(backup_manifest_info *manifest)

@ MANIFEST_OPTION_FORCE_ENCODE

enum manifest_option backup_manifest_option

static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, Oid spcoid, RelFileNumber relfilenumber, unsigned segno, backup_manifest_info *manifest, unsigned num_incremental_blocks, BlockNumber *incremental_blocks, unsigned truncation_block_length)

static const struct exclude_list_item excludeFiles[]

static int compareWalFileNames(const ListCell *a, const ListCell *b)

static void sendFileWithContent(bbsink *sink, const char *filename, const char *content, int len, backup_manifest_info *manifest)

static off_t read_file_data_into_buffer(bbsink *sink, const char *readfilename, int fd, off_t offset, size_t length, BlockNumber blkno, bool verify_checksum, int *checksum_failures)

static void push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx, size_t *bytes_done, void *data, size_t length)

static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, Oid spcoid, IncrementalBackupInfo *ib)

#define SINK_BUFFER_LENGTH

static void parse_basebackup_options(List *options, basebackup_options *opt)

static const char *const excludeDirContents[]

static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf)

void SendBaseBackup(BaseBackupCmd *cmd, IncrementalBackupInfo *ib)

static bool backup_started_in_recovery

static int64 _tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)

static void perform_base_backup(basebackup_options *opt, bbsink *sink, IncrementalBackupInfo *ib)

static int64 sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly, struct backup_manifest_info *manifest, IncrementalBackupInfo *ib)

static bool noverify_checksums

static void _tarWritePadding(bbsink *sink, int len)

static bool verify_page_checksum(Page page, XLogRecPtr start_lsn, BlockNumber blkno, uint16 *expected_checksum)

static ssize_t basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset, const char *filename, bool partial_read_ok)

static long long int total_checksum_failures

bbsink * bbsink_copystream_new(bool send_to_client)

bbsink * bbsink_gzip_new(bbsink *next, pg_compress_specification *compress)

size_t GetIncrementalFileSize(unsigned num_blocks_required)

FileBackupMethod GetFileBackupMethod(IncrementalBackupInfo *ib, const char *path, Oid dboid, Oid spcoid, RelFileNumber relfilenumber, ForkNumber forknum, unsigned segno, size_t size, unsigned *num_blocks_required, BlockNumber *relative_block_numbers, unsigned *truncation_block_length)

void PrepareForIncrementalBackup(IncrementalBackupInfo *ib, BackupState *backup_state)

#define INCREMENTAL_MAGIC

@ BACK_UP_FILE_INCREMENTALLY

bbsink * bbsink_lz4_new(bbsink *next, pg_compress_specification *compress)

void basebackup_progress_wait_checkpoint(void)

bbsink * bbsink_progress_new(bbsink *next, bool estimate_backup_size)

void basebackup_progress_wait_wal_archive(bbsink_state *state)

void basebackup_progress_done(void)

void basebackup_progress_transfer_wal(void)

void basebackup_progress_estimate_backup_size(void)

static void bbsink_begin_backup(bbsink *sink, bbsink_state *state, int buffer_length)

static void bbsink_begin_archive(bbsink *sink, const char *archive_name)

static void bbsink_end_archive(bbsink *sink)

static void bbsink_end_backup(bbsink *sink, XLogRecPtr endptr, TimeLineID endtli)

static void bbsink_cleanup(bbsink *sink)

static void bbsink_archive_contents(bbsink *sink, size_t len)

BaseBackupTargetHandle * BaseBackupGetTargetHandle(char *target, char *target_detail)

bbsink * BaseBackupGetSink(BaseBackupTargetHandle *handle, bbsink *next_sink)

bbsink * bbsink_throttle_new(bbsink *next, uint32 maxrate)

bbsink * bbsink_zstd_new(bbsink *next, pg_compress_specification *compress)

bool parse_bool(const char *value, bool *result)

PageHeaderData * PageHeader

static bool PageIsNew(const PageData *page)

static XLogRecPtr PageGetLSN(const PageData *page)

#define MemSet(start, val, len)

#define StaticAssertDecl(condition, errmessage)

#define StaticAssertStmt(condition, errmessage)

#define OidIsValid(objectId)

uint16 pg_checksum_page(char *page, BlockNumber blkno)

bool pg_checksum_parse_type(char *name, pg_checksum_type *type)

int pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)

int pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)

char * validate_compress_specification(pg_compress_specification *spec)

bool parse_compress_algorithm(char *name, pg_compress_algorithm *algorithm)

void parse_compress_specification(pg_compress_algorithm algorithm, char *specification, pg_compress_specification *result)

char * defGetString(DefElem *def)

bool defGetBoolean(DefElem *def)

int64 defGetInt64(DefElem *def)

int errmsg_plural(const char *fmt_singular, const char *fmt_plural, unsigned long n,...)

int errcode_for_file_access(void)

int errhint(const char *fmt,...)

int errcode(int sqlerrcode)

int errmsg(const char *fmt,...)

#define ereport(elevel,...)

int CloseTransientFile(int fd)

DIR * AllocateDir(const char *dirname)

struct dirent * ReadDir(DIR *dir, const char *dirname)

bool looks_like_temp_rel_name(const char *name)

int OpenTransientFile(const char *fileName, int fileFlags)

#define PG_TEMP_FILE_PREFIX

#define PG_AUTOCONF_FILENAME

Assert(PointerIsAligned(start, uint64))

#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)

#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)

List * lappend(List *list, void *datum)

void list_sort(List *list, list_sort_comparator cmp)

char * pstrdup(const char *in)

void pfree(void *pointer)

void * palloc0(Size size)

#define CHECK_FOR_INTERRUPTS()

#define ERRCODE_DATA_CORRUPTED

static ListCell * lnext(const List *l, const ListCell *c)

void pgstat_prepare_report_checksum_failure(Oid dboid)

void pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount)

static size_t tarPaddingBytesRequired(size_t len)

enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)

char * last_dir_separator(const char *filename)

int pg_strcasecmp(const char *s1, const char *s2)

static Datum BoolGetDatum(bool X)

static int fd(const char *x, int i)

bool update_process_title

static void set_ps_display(const char *activity)

char * psprintf(const char *fmt,...)

bool parse_filename_for_nontemp_relation(const char *name, RelFileNumber *relnumber, ForkNumber *fork, unsigned *segno)

#define RELCACHE_INIT_FILENAME

#define InvalidRelFileNumber

#define TABLESPACE_VERSION_DIRECTORY

#define RelFileNumberIsValid(relnumber)

void ReleaseAuxProcessResources(bool isCommit)

ResourceOwner CurrentResourceOwner

ResourceOwner AuxProcessResourceOwner

void destroyStringInfo(StringInfo str)

StringInfo makeStringInfo(void)

pg_compress_specification compression_specification

pg_checksum_type manifest_checksum_type

pg_compress_algorithm compression

backup_manifest_option manifest

BaseBackupTargetHandle * target_handle

pg_compress_algorithm algorithm

#define LOG_METAINFO_DATAFILE_TMP

static void pgstat_report_wait_start(uint32 wait_event_info)

static void pgstat_report_wait_end(void)

void WalSndSetState(WalSndState state)

#define readlink(path, buf, size)

bool RecoveryInProgress(void)

void do_pg_abort_backup(int code, Datum arg)

SessionBackupState get_backup_status(void)

void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)

bool DataChecksumsEnabled(void)

void do_pg_backup_start(const char *backupidstr, bool fast, List **tablespaces, BackupState *state, StringInfo tblspcmapfile)

void do_pg_backup_stop(BackupState *state, bool waitforarchive)

#define BACKUP_LABEL_FILE

#define XLOG_CONTROL_FILE

static bool IsXLogFileName(const char *fname)

static void XLogFromFileName(const char *fname, TimeLineID *tli, XLogSegNo *logSegNo, int wal_segsz_bytes)

#define XLByteToPrevSeg(xlrp, logSegNo, wal_segsz_bytes)

static bool IsTLHistoryFileName(const char *fname)

static void StatusFilePath(char *path, const char *xlog, const char *suffix)

#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)

static void XLogFileName(char *fname, TimeLineID tli, XLogSegNo logSegNo, int wal_segsz_bytes)

char * build_backup_content(BackupState *state, bool ishistoryfile)

static BackupState * backup_state

static StringInfo tablespace_map