Skip to content

Commit 4a1842b

Browse files
author
Oleg Gurev
committed
Merge remote-tracking branch 'origin/ce' into merge-ce
2 parents 166c3dc + 4fd3087 commit 4a1842b

File tree

2 files changed

+390
-1
lines changed

2 files changed

+390
-1
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ It is designed to allow false positives (i.e. block/page is marked in the `ptrac
1212

1313
Currently, `ptrack` codebase is split between small PostgreSQL core patch and extension. All public SQL API methods and main engine are placed in the `ptrack` extension, while the core patch contains only certain hooks and modifies binary utilities to ignore `ptrack.map.*` files.
1414

15-
This extension is compatible with PostgreSQL [11](patches/REL_11_STABLE-ptrack-core.diff), [12](patches/REL_12_STABLE-ptrack-core.diff), [13](patches/REL_13_STABLE-ptrack-core.diff), [14](patches/REL_14_STABLE-ptrack-core.diff), [15](patches/REL_15_STABLE-ptrack-core.diff), [16](patches/REL_16_STABLE-ptrack-core.diff), [17](patches/REL_17_STABLE-ptrack-core.diff).
15+
This extension is compatible with PostgreSQL [11](patches/REL_11_STABLE-ptrack-core.diff), [12](patches/REL_12_STABLE-ptrack-core.diff), [13](patches/REL_13_STABLE-ptrack-core.diff), [14](patches/REL_14_STABLE-ptrack-core.diff), [15](patches/REL_15_STABLE-ptrack-core.diff), [16](patches/REL_16_STABLE-ptrack-core.diff), [17](patches/REL_17_STABLE-ptrack-core.diff), [18](patches/REL_18_STABLE-ptrack-core.diff).
1616

1717
## Installation
1818

‎patches/REL_18_STABLE-ptrack-core.diff‎

Lines changed: 389 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,389 @@
1+
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
2+
index 596d2ca5836..3a72b8d2bf4 100644
3+
--- a/src/backend/access/transam/xlog.c
4+
+++ b/src/backend/access/transam/xlog.c
5+
@@ -136,6 +136,7 @@ int wal_retrieve_retry_interval = 5000;
6+
int max_slot_wal_keep_size_mb = -1;
7+
int wal_decode_buffer_size = 512 * 1024;
8+
bool track_wal_io_timing = false;
9+
+backup_checkpoint_request_hook_type backup_checkpoint_request_hook = NULL;
10+
11+
#ifdef WAL_DEBUG
12+
bool XLOG_DEBUG = false;
13+
@@ -8929,6 +8930,12 @@ do_pg_backup_start(const char *backupidstr, bool fast, List **tablespaces,
14+
{
15+
bool checkpointfpw;
16+
17+
+ /*
18+
+ * Before we call RequestCheckpoint() we need to set
19+
+ * init_lsn for ptrack map
20+
+ */
21+
+ if (backup_checkpoint_request_hook)
22+
+ backup_checkpoint_request_hook();
23+
/*
24+
* Force a CHECKPOINT. Aside from being necessary to prevent torn
25+
* page problems, this guarantees that two successive backup runs
26+
diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c
27+
index f0f88838dc2..239d50be357 100644
28+
--- a/src/backend/backup/basebackup.c
29+
+++ b/src/backend/backup/basebackup.c
30+
@@ -219,6 +219,12 @@ static const struct exclude_list_item excludeFiles[] =
31+
32+
{"postmaster.pid", false},
33+
{"postmaster.opts", false},
34+
+ /*
35+
+ * Skip all transient ptrack files, but do copy ptrack.map, since it may
36+
+ * be successfully used immediately after backup. TODO: check, test?
37+
+ */
38+
+ {"ptrack.map.mmap", false},
39+
+ {"ptrack.map.tmp", false},
40+
41+
/* end of list */
42+
{NULL, false}
43+
diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c
44+
index aa8c64a2c9e..56ed1b4df0a 100644
45+
--- a/src/backend/storage/file/copydir.c
46+
+++ b/src/backend/storage/file/copydir.c
47+
@@ -30,6 +30,8 @@
48+
#include "storage/copydir.h"
49+
#include "storage/fd.h"
50+
51+
+copydir_hook_type copydir_hook = NULL;
52+
+
53+
/* GUCs */
54+
int file_copy_method = FILE_COPY_METHOD_COPY;
55+
56+
@@ -91,6 +93,9 @@ copydir(const char *fromdir, const char *todir, bool recurse)
57+
}
58+
FreeDir(xldir);
59+
60+
+ if (copydir_hook)
61+
+ copydir_hook(todir);
62+
+
63+
/*
64+
* Be paranoid here and fsync all files to ensure the copy is really done.
65+
* But if fsync is disabled, we're done.
66+
diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c
67+
index 2ccb0faceb5..ad08c0c6960 100644
68+
--- a/src/backend/storage/smgr/md.c
69+
+++ b/src/backend/storage/smgr/md.c
70+
@@ -86,6 +86,8 @@ typedef struct _MdfdVec
71+
72+
static MemoryContext MdCxt; /* context for all MdfdVec objects */
73+
74+
+mdextend_hook_type mdextend_hook = NULL;
75+
+mdwrite_hook_type mdwrite_hook = NULL;
76+
77+
/* Populate a file tag describing an md.c segment file. */
78+
#define INIT_MD_FILETAG(a,xx_rlocator,xx_forknum,xx_segno) \
79+
@@ -530,6 +532,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum,
80+
register_dirty_segment(reln, forknum, v);
81+
82+
Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE));
83+
+
84+
+ if (mdextend_hook)
85+
+ mdextend_hook(reln->smgr_rlocator, forknum, blocknum);
86+
}
87+
88+
/*
89+
@@ -637,6 +642,12 @@ mdzeroextend(SMgrRelation reln, ForkNumber forknum,
90+
91+
remblocks -= numblocks;
92+
curblocknum += numblocks;
93+
+
94+
+ if (mdextend_hook)
95+
+ {
96+
+ for (; blocknum < curblocknum; blocknum++)
97+
+ mdextend_hook(reln->smgr_rlocator, forknum, blocknum);
98+
+ }
99+
}
100+
}
101+
102+
@@ -1139,7 +1150,14 @@ mdwritev(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum,
103+
104+
nblocks -= nblocks_this_segment;
105+
buffers += nblocks_this_segment;
106+
- blocknum += nblocks_this_segment;
107+
+
108+
+ if (mdwrite_hook)
109+
+ {
110+
+ for (; nblocks_this_segment--; blocknum++)
111+
+ mdwrite_hook(reln->smgr_rlocator, forknum, blocknum);
112+
+ }
113+
+ else
114+
+ blocknum += nblocks_this_segment;
115+
}
116+
}
117+
118+
diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c
119+
index fc16db90133..08cd553ad55 100644
120+
--- a/src/backend/storage/sync/sync.c
121+
+++ b/src/backend/storage/sync/sync.c
122+
@@ -74,6 +74,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */
123+
static CycleCtr sync_cycle_ctr = 0;
124+
static CycleCtr checkpoint_cycle_ctr = 0;
125+
126+
+ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL;
127+
+
128+
/* Intervals for calling AbsorbSyncRequests */
129+
#define FSYNCS_PER_ABSORB 10
130+
#define UNLINKS_PER_ABSORB 10
131+
@@ -470,6 +472,9 @@ ProcessSyncRequests(void)
132+
CheckpointStats.ckpt_longest_sync = longest;
133+
CheckpointStats.ckpt_agg_sync_time = total_elapsed;
134+
135+
+ if (ProcessSyncRequests_hook)
136+
+ ProcessSyncRequests_hook();
137+
+
138+
/* Flag successful completion of ProcessSyncRequests */
139+
sync_in_progress = false;
140+
}
141+
diff --git a/src/backend/utils/init/miscinit.c b/src/backend/utils/init/miscinit.c
142+
index df182577a19..8e4d7526db1 100644
143+
--- a/src/backend/utils/init/miscinit.c
144+
+++ b/src/backend/utils/init/miscinit.c
145+
@@ -80,6 +80,11 @@ static Latch LocalLatchData;
146+
147+
bool IgnoreSystemIndexes = false;
148+
149+
+/* ----------------------------------------------------------------
150+
+ * Ptrack functions support
151+
+ * ----------------------------------------------------------------
152+
+ */
153+
+static void check_use_ptrack(const char *libraries);
154+
155+
/* ----------------------------------------------------------------
156+
* common process startup code
157+
@@ -1908,6 +1913,8 @@ process_shared_preload_libraries(void)
158+
false);
159+
process_shared_preload_libraries_in_progress = false;
160+
process_shared_preload_libraries_done = true;
161+
+
162+
+ check_use_ptrack(shared_preload_libraries_string);
163+
}
164+
165+
/*
166+
@@ -1950,3 +1957,71 @@ pg_bindtextdomain(const char *domain)
167+
}
168+
#endif
169+
}
170+
+
171+
+/*-------------------------------------------------------------------------
172+
+ * Ptrack functions support
173+
+ *-------------------------------------------------------------------------
174+
+ */
175+
+
176+
+/* Persistent copy of ptrack.map to restore after crash */
177+
+#define PTRACK_PATH "global/ptrack.map"
178+
+/* Used for atomical crash-safe update of ptrack.map */
179+
+#define PTRACK_PATH_TMP "global/ptrack.map.tmp"
180+
+
181+
+/*
182+
+ * Check that path is accessible by us and return true if it is
183+
+ * not a directory.
184+
+ */
185+
+static bool
186+
+ptrack_file_exists(const char *path)
187+
+{
188+
+ struct stat st;
189+
+
190+
+ Assert(path != NULL);
191+
+
192+
+ if (stat(path, &st) == 0)
193+
+ return S_ISDIR(st.st_mode) ? false : true;
194+
+ else if (!(errno == ENOENT || errno == ENOTDIR || errno == EACCES))
195+
+ ereport(ERROR,
196+
+ (errcode_for_file_access(),
197+
+ errmsg("could not access file \"%s\": %m", path)));
198+
+
199+
+ return false;
200+
+}
201+
+
202+
+/*
203+
+ * Delete ptrack files when ptrack is disabled.
204+
+ *
205+
+ * This is performed by postmaster at start,
206+
+ * so that there are no concurrent delete issues.
207+
+ */
208+
+static void
209+
+ptrackCleanFiles(void)
210+
+{
211+
+ char ptrack_path[MAXPGPATH];
212+
+ char ptrack_path_tmp[MAXPGPATH];
213+
+
214+
+ sprintf(ptrack_path, "%s/%s", DataDir, PTRACK_PATH);
215+
+ sprintf(ptrack_path_tmp, "%s/%s", DataDir, PTRACK_PATH_TMP);
216+
+
217+
+ elog(DEBUG1, "ptrack: clean map files");
218+
+
219+
+ if (ptrack_file_exists(ptrack_path_tmp))
220+
+ durable_unlink(ptrack_path_tmp, LOG);
221+
+
222+
+ if (ptrack_file_exists(ptrack_path))
223+
+ durable_unlink(ptrack_path, LOG);
224+
+}
225+
+
226+
+static void
227+
+check_use_ptrack(const char *libraries)
228+
+{
229+
+ /* We need to delete the ptrack.map file when ptrack was turned off */
230+
+ if (strstr(shared_preload_libraries_string, "ptrack") == NULL)
231+
+ {
232+
+ ptrackCleanFiles();
233+
+ }
234+
+}
235+
+
236+
+#undef PTRACK_PATH
237+
+#undef PTRACK_PATH_TMP
238+
diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c
239+
index f20be82862a..84c1d82b9cd 100644
240+
--- a/src/bin/pg_checksums/pg_checksums.c
241+
+++ b/src/bin/pg_checksums/pg_checksums.c
242+
@@ -109,6 +109,11 @@ static const struct exclude_list_item skip[] = {
243+
{"pg_filenode.map", false},
244+
{"pg_internal.init", true},
245+
{"PG_VERSION", false},
246+
+
247+
+ {"ptrack.map.mmap", false},
248+
+ {"ptrack.map", false},
249+
+ {"ptrack.map.tmp", false},
250+
+
251+
#ifdef EXEC_BACKEND
252+
{"config_exec_params", true},
253+
#endif
254+
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
255+
index e876f35f38e..1fa3bf4d222 100644
256+
--- a/src/bin/pg_resetwal/pg_resetwal.c
257+
+++ b/src/bin/pg_resetwal/pg_resetwal.c
258+
@@ -87,6 +87,7 @@ static void FindEndOfXLOG(void);
259+
static void KillExistingXLOG(void);
260+
static void KillExistingArchiveStatus(void);
261+
static void KillExistingWALSummaries(void);
262+
+static void KillExistingPtrack(void);
263+
static void WriteEmptyXLOG(void);
264+
static void usage(void);
265+
266+
@@ -517,6 +518,7 @@ main(int argc, char *argv[])
267+
KillExistingXLOG();
268+
KillExistingArchiveStatus();
269+
KillExistingWALSummaries();
270+
+ KillExistingPtrack();
271+
WriteEmptyXLOG();
272+
273+
printf(_("Write-ahead log reset\n"));
274+
@@ -1022,6 +1024,40 @@ KillExistingXLOG(void)
275+
pg_fatal("could not close directory \"%s\": %m", XLOGDIR);
276+
}
277+
278+
+/*
279+
+ * Remove existing ptrack files
280+
+ */
281+
+static void
282+
+KillExistingPtrack(void)
283+
+{
284+
+#define PTRACKDIR "global"
285+
+
286+
+ DIR *xldir;
287+
+ struct dirent *xlde;
288+
+ char path[MAXPGPATH + sizeof(PTRACKDIR)];
289+
+
290+
+ xldir = opendir(PTRACKDIR);
291+
+ if (xldir == NULL)
292+
+ pg_fatal("could not open directory \"%s\": %m", PTRACKDIR);
293+
+
294+
+ while (errno = 0, (xlde = readdir(xldir)) != NULL)
295+
+ {
296+
+ if (strcmp(xlde->d_name, "ptrack.map.mmap") == 0 ||
297+
+ strcmp(xlde->d_name, "ptrack.map") == 0 ||
298+
+ strcmp(xlde->d_name, "ptrack.map.tmp") == 0)
299+
+ {
300+
+ snprintf(path, sizeof(path), "%s/%s", PTRACKDIR, xlde->d_name);
301+
+ if (unlink(path) < 0)
302+
+ pg_fatal("could not delete file \"%s\": %m", path);
303+
+ }
304+
+ }
305+
+
306+
+ if (errno)
307+
+ pg_fatal("could not read directory \"%s\": %m", PTRACKDIR);
308+
+
309+
+ if (closedir(xldir))
310+
+ pg_fatal("could not close directory \"%s\": %m", PTRACKDIR);
311+
+}
312+
313+
/*
314+
* Remove existing archive status files
315+
diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c
316+
index c933871ca9f..f34c0edf350 100644
317+
--- a/src/bin/pg_rewind/filemap.c
318+
+++ b/src/bin/pg_rewind/filemap.c
319+
@@ -186,6 +186,10 @@ static const struct exclude_list_item excludeFiles[] =
320+
{"postmaster.pid", false},
321+
{"postmaster.opts", false},
322+
323+
+ {"ptrack.map.mmap", false},
324+
+ {"ptrack.map", false},
325+
+ {"ptrack.map.tmp", false},
326+
+
327+
/* end of list */
328+
{NULL, false}
329+
};
330+
diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
331+
index adddac6710e..d57d04a6fd5 100644
332+
--- a/src/include/access/xlog.h
333+
+++ b/src/include/access/xlog.h
334+
@@ -59,6 +59,9 @@ extern PGDLLIMPORT int wal_decode_buffer_size;
335+
336+
extern PGDLLIMPORT int CheckPointSegments;
337+
338+
+typedef void (*backup_checkpoint_request_hook_type) (void);
339+
+extern PGDLLIMPORT backup_checkpoint_request_hook_type backup_checkpoint_request_hook;
340+
+
341+
/* Archive modes */
342+
typedef enum ArchiveMode
343+
{
344+
diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h
345+
index f1d7beeed1a..f162c4405dc 100644
346+
--- a/src/include/storage/copydir.h
347+
+++ b/src/include/storage/copydir.h
348+
@@ -22,6 +22,9 @@ typedef enum FileCopyMethod
349+
/* GUC parameters */
350+
extern PGDLLIMPORT int file_copy_method;
351+
352+
+typedef void (*copydir_hook_type) (const char *path);
353+
+extern PGDLLIMPORT copydir_hook_type copydir_hook;
354+
+
355+
extern void copydir(const char *fromdir, const char *todir, bool recurse);
356+
extern void copy_file(const char *fromfile, const char *tofile);
357+
358+
diff --git a/src/include/storage/md.h b/src/include/storage/md.h
359+
index b563c27abf0..29fbc1c80c3 100644
360+
--- a/src/include/storage/md.h
361+
+++ b/src/include/storage/md.h
362+
@@ -22,6 +22,13 @@
363+
364+
extern PGDLLIMPORT const PgAioHandleCallbacks aio_md_readv_cb;
365+
366+
+typedef void (*mdextend_hook_type) (RelFileLocatorBackend smgr_rlocator,
367+
+ ForkNumber forknum, BlockNumber blocknum);
368+
+extern PGDLLIMPORT mdextend_hook_type mdextend_hook;
369+
+typedef void (*mdwrite_hook_type) (RelFileLocatorBackend smgr_rlocator,
370+
+ ForkNumber forknum, BlockNumber blocknum);
371+
+extern PGDLLIMPORT mdwrite_hook_type mdwrite_hook;
372+
+
373+
/* md storage manager functionality */
374+
extern void mdinit(void);
375+
extern void mdopen(SMgrRelation reln);
376+
diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h
377+
index c2272d14175..4132cfd92c6 100644
378+
--- a/src/include/storage/sync.h
379+
+++ b/src/include/storage/sync.h
380+
@@ -55,6 +55,9 @@ typedef struct FileTag
381+
uint64 segno;
382+
} FileTag;
383+
384+
+typedef void (*ProcessSyncRequests_hook_type) (void);
385+
+extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook;
386+
+
387+
extern void InitSync(void);
388+
extern void SyncPreCheckpoint(void);
389+
extern void SyncPostCheckpoint(void);

0 commit comments

Comments
 (0)