Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 5028f22

Browse filesBrowse files
hlinnakahlinnaka
committed
Switch to CRC-32C in WAL and other places.
The old algorithm was found to not be the usual CRC-32 algorithm, used by Ethernet et al. We were using a non-reflected lookup table with code meant for a reflected lookup table. That's a strange combination that AFAICS does not correspond to any bit-wise CRC calculation, which makes it difficult to reason about its properties. Although it has worked well in practice, seems safer to use a well-known algorithm. Since we're changing the algorithm anyway, we might as well choose a different polynomial. The Castagnoli polynomial has better error-correcting properties than the traditional CRC-32 polynomial, even if we had implemented it correctly. Another reason for picking that is that some new CPUs have hardware support for calculating CRC-32C, but not CRC-32, let alone our strange variant of it. This patch doesn't add any support for such hardware, but a future patch could now do that. The old algorithm is kept around for tsquery and pg_trgm, which use the values in indexes that need to remain compatible so that pg_upgrade works. While we're at it, share the old lookup table for CRC-32 calculation between hstore, ltree and core. They all use the same table, so might as well.
1 parent 404bc51 commit 5028f22
Copy full SHA for 5028f22

File tree

Expand file treeCollapse file tree

20 files changed

+300
-341
lines changed
Filter options
Expand file treeCollapse file tree

20 files changed

+300
-341
lines changed

‎contrib/hstore/Makefile

Copy file name to clipboardExpand all lines: contrib/hstore/Makefile
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
MODULE_big = hstore
44
OBJS = hstore_io.o hstore_op.o hstore_gist.o hstore_gin.o hstore_compat.o \
5-
crc32.o $(WIN32RES)
5+
$(WIN32RES)
66

77
EXTENSION = hstore
88
DATA = hstore--1.3.sql hstore--1.2--1.3.sql \

‎contrib/hstore/crc32.c

Copy file name to clipboardExpand all lines: contrib/hstore/crc32.c
-106Lines changed: 0 additions & 106 deletions
This file was deleted.

‎contrib/hstore/crc32.h

Copy file name to clipboardExpand all lines: contrib/hstore/crc32.h
-13Lines changed: 0 additions & 13 deletions
This file was deleted.

‎contrib/hstore/hstore_gist.c

Copy file name to clipboardExpand all lines: contrib/hstore/hstore_gist.c
+15-1Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
#include "access/gist.h"
77
#include "access/skey.h"
88
#include "catalog/pg_type.h"
9+
#include "utils/pg_crc.h"
910

10-
#include "crc32.h"
1111
#include "hstore.h"
1212

1313
/* bigint defines */
@@ -68,6 +68,20 @@ typedef struct
6868

6969
#define WISH_F(a,b,c) (double)( -(double)(((a)-(b))*((a)-(b))*((a)-(b)))*(c) )
7070

71+
/* shorthand for calculating CRC-32 of a single chunk of data. */
72+
static pg_crc32
73+
crc32_sz(char *buf, int size)
74+
{
75+
pg_crc32 crc;
76+
77+
INIT_TRADITIONAL_CRC32(crc);
78+
COMP_TRADITIONAL_CRC32(crc, buf, size);
79+
FIN_TRADITIONAL_CRC32(crc);
80+
81+
return crc;
82+
}
83+
84+
7185
PG_FUNCTION_INFO_V1(ghstore_in);
7286
PG_FUNCTION_INFO_V1(ghstore_out);
7387

‎contrib/ltree/crc32.c

Copy file name to clipboard
+21-93Lines changed: 21 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
1-
/* Both POSIX and CRC32 checksums */
2-
31
/* contrib/ltree/crc32.c */
42

3+
/*
4+
* Implements CRC-32, as used in ltree.
5+
*
6+
* Note that the CRC is used in the on-disk format of GiST indexes, so we
7+
* must stay backwards-compatible!
8+
*/
9+
510
#include "postgres.h"
611

712
#include <sys/types.h>
@@ -15,100 +20,23 @@
1520
#define TOLOWER(x) (x)
1621
#endif
1722

23+
#include "utils/pg_crc.h"
1824
#include "crc32.h"
1925

20-
/*
21-
* This code implements the AUTODIN II polynomial
22-
* The variable corresponding to the macro argument "crc" should
23-
* be an unsigned long.
24-
* Oroginal code by Spencer Garrett <srg@quick.com>
25-
*/
26-
27-
#define _CRC32_(crc, ch) ((crc) = ((crc) >> 8) ^ crc32tab[((crc) ^ (ch)) & 0xff])
28-
29-
/* generated using the AUTODIN II polynomial
30-
* x^32 + x^26 + x^23 + x^22 + x^16 +
31-
* x^12 + x^11 + x^10 + x^8 + x^7 + x^5 + x^4 + x^2 + x^1 + 1
32-
*/
33-
34-
static const unsigned int crc32tab[256] = {
35-
0x00000000, 0x77073096, 0xee0e612c, 0x990951ba,
36-
0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
37-
0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
38-
0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
39-
0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
40-
0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
41-
0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,
42-
0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
43-
0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
44-
0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
45-
0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940,
46-
0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
47-
0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116,
48-
0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
49-
0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
50-
0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
51-
0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a,
52-
0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
53-
0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818,
54-
0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
55-
0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
56-
0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
57-
0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c,
58-
0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
59-
0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
60-
0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
61-
0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
62-
0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
63-
0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086,
64-
0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
65-
0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4,
66-
0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
67-
0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
68-
0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
69-
0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
70-
0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
71-
0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe,
72-
0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
73-
0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
74-
0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
75-
0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252,
76-
0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
77-
0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60,
78-
0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
79-
0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
80-
0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
81-
0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04,
82-
0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
83-
0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a,
84-
0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
85-
0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
86-
0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
87-
0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e,
88-
0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
89-
0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
90-
0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
91-
0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
92-
0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
93-
0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0,
94-
0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
95-
0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6,
96-
0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
97-
0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
98-
0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d,
99-
};
100-
10126
unsigned int
10227
ltree_crc32_sz(char *buf, int size)
10328
{
104-
unsigned int crc = ~((unsigned int) 0);
105-
char *p;
106-
int len,
107-
nr;
108-
109-
len = 0;
110-
nr = size;
111-
for (len += nr, p = buf; nr--; ++p)
112-
_CRC32_(crc, TOLOWER((unsigned int) *p));
113-
return ~crc;
29+
pg_crc32 crc;
30+
char *p = buf;
31+
32+
INIT_TRADITIONAL_CRC32(crc);
33+
while (size > 0)
34+
{
35+
char c = (char) TOLOWER(*p);
36+
COMP_TRADITIONAL_CRC32(crc, &c, 1);
37+
size--;
38+
p++;
39+
}
40+
FIN_TRADITIONAL_CRC32(crc);
41+
return (unsigned int) crc;
11442
}

‎contrib/pg_trgm/trgm_op.c

Copy file name to clipboardExpand all lines: contrib/pg_trgm/trgm_op.c
+3-3Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,9 @@ compact_trigram(trgm *tptr, char *str, int bytelen)
108108
{
109109
pg_crc32 crc;
110110

111-
INIT_CRC32(crc);
112-
COMP_CRC32(crc, str, bytelen);
113-
FIN_CRC32(crc);
111+
INIT_LEGACY_CRC32(crc);
112+
COMP_LEGACY_CRC32(crc, str, bytelen);
113+
FIN_LEGACY_CRC32(crc);
114114

115115
/*
116116
* use only 3 upper bytes from crc, hope, it's good enough hashing

‎src/backend/access/transam/twophase.c

Copy file name to clipboardExpand all lines: src/backend/access/transam/twophase.c
+12-12Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -847,9 +847,9 @@ TwoPhaseGetDummyProc(TransactionId xid)
847847
* 6. TwoPhaseRecordOnDisk
848848
* 7. ...
849849
* 8. TwoPhaseRecordOnDisk (end sentinel, rmid == TWOPHASE_RM_END_ID)
850-
* 9. CRC32
850+
* 9. checksum (CRC-32C)
851851
*
852-
* Each segment except the final CRC32 is MAXALIGN'd.
852+
* Each segment except the final checksum is MAXALIGN'd.
853853
*/
854854

855855
/*
@@ -1056,11 +1056,11 @@ EndPrepare(GlobalTransaction gxact)
10561056
path)));
10571057

10581058
/* Write data to file, and calculate CRC as we pass over it */
1059-
INIT_CRC32(statefile_crc);
1059+
INIT_CRC32C(statefile_crc);
10601060

10611061
for (record = records.head; record != NULL; record = record->next)
10621062
{
1063-
COMP_CRC32(statefile_crc, record->data, record->len);
1063+
COMP_CRC32C(statefile_crc, record->data, record->len);
10641064
if ((write(fd, record->data, record->len)) != record->len)
10651065
{
10661066
CloseTransientFile(fd);
@@ -1070,7 +1070,7 @@ EndPrepare(GlobalTransaction gxact)
10701070
}
10711071
}
10721072

1073-
FIN_CRC32(statefile_crc);
1073+
FIN_CRC32C(statefile_crc);
10741074

10751075
/*
10761076
* Write a deliberately bogus CRC to the state file; this is just paranoia
@@ -1289,13 +1289,13 @@ ReadTwoPhaseFile(TransactionId xid, bool give_warnings)
12891289
return NULL;
12901290
}
12911291

1292-
INIT_CRC32(calc_crc);
1293-
COMP_CRC32(calc_crc, buf, crc_offset);
1294-
FIN_CRC32(calc_crc);
1292+
INIT_CRC32C(calc_crc);
1293+
COMP_CRC32C(calc_crc, buf, crc_offset);
1294+
FIN_CRC32C(calc_crc);
12951295

12961296
file_crc = *((pg_crc32 *) (buf + crc_offset));
12971297

1298-
if (!EQ_CRC32(calc_crc, file_crc))
1298+
if (!EQ_CRC32C(calc_crc, file_crc))
12991299
{
13001300
pfree(buf);
13011301
return NULL;
@@ -1540,9 +1540,9 @@ RecreateTwoPhaseFile(TransactionId xid, void *content, int len)
15401540
int fd;
15411541

15421542
/* Recompute CRC */
1543-
INIT_CRC32(statefile_crc);
1544-
COMP_CRC32(statefile_crc, content, len);
1545-
FIN_CRC32(statefile_crc);
1543+
INIT_CRC32C(statefile_crc);
1544+
COMP_CRC32C(statefile_crc, content, len);
1545+
FIN_CRC32C(statefile_crc);
15461546

15471547
TwoPhaseFilePath(path, xid);
15481548

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.