Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
# NEWS for rsync 4.0.0 (UNRELEASED)

## Enhancements

- Added SHA-3 (FIPS 202) transfer checksum choices `sha3-256`,
`sha3-384` and `sha3-512` for builds with OpenSSL SHA-3 support.
Default negotiation is unchanged. Select SHA-3 with
`--checksum-choice` or include it in both peers' `RSYNC_CHECKSUM_LIST`.

## Bug fixes

- Fixed `-X` transfers of user xattrs on NetBSD by using the native BSD
Expand Down
24 changes: 20 additions & 4 deletions checksum.c
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,6 @@ extern int protocol_version;
extern int proper_seed_order;
extern const char *checksum_choice;

#define NNI_BUILTIN (1<<0)
#define NNI_EVP (1<<1)
#define NNI_EVP_OK (1<<2)

struct name_num_item valid_checksums_items[] = {
#ifdef SUPPORT_XXH3
{ CSUM_XXH3_128, 0, "xxh128", NULL },
Expand All @@ -55,6 +51,15 @@ struct name_num_item valid_checksums_items[] = {
#ifdef SUPPORT_XXHASH
{ CSUM_XXH64, 0, "xxh64", NULL },
{ CSUM_XXH64, 0, "xxhash", NULL },
#endif
#ifdef USE_OPENSSL
/* SHA-3 is offered only via OpenSSL EVP (sha3-* digests, OpenSSL >= 1.1.1).
* verify_digest() probes each entry at startup and disables any that the
* linked libcrypto does not support, so listing them here is safe even
* when the build environment is older. */
{ CSUM_SHA3_512, NNI_EVP|NNI_EXPLICIT_ONLY, "sha3-512", NULL },
{ CSUM_SHA3_384, NNI_EVP|NNI_EXPLICIT_ONLY, "sha3-384", NULL },
{ CSUM_SHA3_256, NNI_EVP|NNI_EXPLICIT_ONLY, "sha3-256", NULL },
#endif
{ CSUM_MD5, NNI_BUILTIN|NNI_EVP, "md5", NULL },
{ CSUM_MD4, NNI_BUILTIN|NNI_EVP, "md4", NULL },
Expand Down Expand Up @@ -256,6 +261,14 @@ int csum_len_for_type(int cst, BOOL flist_csum)
#ifdef SHA512_DIGEST_LENGTH
case CSUM_SHA512:
return SHA512_DIGEST_LENGTH;
#endif
#ifdef USE_OPENSSL
case CSUM_SHA3_256:
return SHA3_256_DIGEST_LEN;
case CSUM_SHA3_384:
return SHA3_384_DIGEST_LEN;
case CSUM_SHA3_512:
return SHA3_512_DIGEST_LEN;
#endif
case CSUM_XXH64:
case CSUM_XXH3_64:
Expand Down Expand Up @@ -285,6 +298,9 @@ int canonical_checksum(int csum_type)
case CSUM_SHA1:
case CSUM_SHA256:
case CSUM_SHA512:
case CSUM_SHA3_256:
case CSUM_SHA3_384:
case CSUM_SHA3_512:
return -1;
case CSUM_XXH64:
case CSUM_XXH3_64:
Expand Down
5 changes: 3 additions & 2 deletions compat.c
Original file line number Diff line number Diff line change
Expand Up @@ -459,7 +459,8 @@ void validate_choice_vs_env(int ntype, int num1, int num2)
* for the order of the args in the array. If dup_markup == '\0', duplicates
* are removed otherwise the char is prefixed to the duplicate term and, if it
* is an opening paren/bracket/brace, the matching closing char is suffixed.
* "none" is removed on the client side unless dup_markup != '\0'. */
* "none" is removed on the client side unless dup_markup != '\0'.
* Negotiation lists omit explicit-only choices. Display lists keep them. */
int get_default_nno_list(struct name_num_obj *nno, char *to_buf, int to_buf_len, char dup_markup)
{
struct name_num_item *nni;
Expand All @@ -476,7 +477,7 @@ int get_default_nno_list(struct name_num_obj *nno, char *to_buf, int to_buf_len,
init_nno_saw(nno, 0);

for (nni = nno->list, len = 0; nni->name; nni++) {
if (nni->num == CSUM_gone)
if (nni->num == CSUM_gone || (!dup_markup && (nni->flags & NNI_EXPLICIT_ONLY)))
continue;
if (nni->main_nni) {
if (!dup_markup || nni->main_nni->num == CSUM_gone)
Expand Down
14 changes: 13 additions & 1 deletion lib/md-defines.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,16 @@

#define MD4_DIGEST_LEN 16
#define MD5_DIGEST_LEN 16
#if defined SHA512_DIGEST_LENGTH

/* OpenSSL exposes SHA-3 via EVP without SHA3_*_DIGEST_LENGTH macros. */
#define SHA3_256_DIGEST_LEN 32
#define SHA3_384_DIGEST_LEN 48
#define SHA3_512_DIGEST_LEN 64

#if defined USE_OPENSSL
/* SHA3-512 can be used even when SHA-512 is disabled. */
#define MAX_DIGEST_LEN SHA3_512_DIGEST_LEN
#elif defined SHA512_DIGEST_LENGTH
#define MAX_DIGEST_LEN SHA512_DIGEST_LENGTH
#elif defined SHA256_DIGEST_LENGTH
#define MAX_DIGEST_LEN SHA256_DIGEST_LENGTH
Expand All @@ -35,3 +44,6 @@
#define CSUM_SHA1 9
#define CSUM_SHA256 10
#define CSUM_SHA512 11
#define CSUM_SHA3_256 12
#define CSUM_SHA3_384 13
#define CSUM_SHA3_512 14
11 changes: 9 additions & 2 deletions rsync.1.md
Original file line number Diff line number Diff line change
Expand Up @@ -2037,13 +2037,20 @@ sign) if you want the local shell to expand it.
- `xxh128`
- `xxh3`
- `xxh64` (aka `xxhash`)
- `sha3-512`
- `sha3-384`
- `sha3-256`
- `md5`
- `md4`
- `sha1`
- `none`

Run `rsync --version` to see the default checksum list compiled into your
version (which may differ from the list above).
The `sha3-*` choices require an rsync built with OpenSSL support and
a libcrypto that provides SHA-3. They are not offered during default
negotiation. Use `--checksum-choice` or include them in both peers'
[`RSYNC_CHECKSUM_LIST`](#).

Run `rsync --version` to see the checksum choices available in your build.

If "none" is specified for the first (or only) name, the [`--whole-file`](#opt)
option is forced on and no checksum verification is performed on the
Expand Down
5 changes: 5 additions & 0 deletions rsync.h
Original file line number Diff line number Diff line change
Expand Up @@ -1235,6 +1235,11 @@ typedef struct {
#define NSTR_CHECKSUM 0
#define NSTR_COMPRESS 1

#define NNI_BUILTIN (1<<0)
#define NNI_EVP (1<<1)
#define NNI_EVP_OK (1<<2)
#define NNI_EXPLICIT_ONLY (1<<3)

struct name_num_item {
int num, flags;
const char *name;
Expand Down
123 changes: 123 additions & 0 deletions testsuite/tests/transfer/checksum-sha3.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
#!/usr/bin/env python3
# SHA-3 checksum support (sha3-256, sha3-384, sha3-512).
#
import json
import os
from random import Random
import re
import subprocess

from harness.environment import FROMDIR, TODIR
from harness.filesystem import assert_same, makepath, rmtree, walk_files
from harness.results import test_fail
from harness.rsync import forced_protocol, rsync_argv, run_rsync
from harness import metadata

metadata(transports={'pipe'}, mutates={'environment', 'filesystem'}, tags={'transfer', 'regression'})

ALGOS = ('sha3-256', 'sha3-384', 'sha3-512')

choices = json.loads(run_rsync('-VV', capture_output=True).stdout)['checksum_list']
available = [algo for algo in ALGOS if algo in choices]
os.environ.pop('RSYNC_CHECKSUM_LIST', None)
protocol = forced_protocol()
legacy_checksum = protocol is not None and protocol < 30

rmtree(FROMDIR)
makepath(FROMDIR / 'sub')
# Empty, small and chunk-spanning sizes (CHUNK_SIZE is 32k internally).
(FROMDIR / 'empty').write_bytes(b'')
(FROMDIR / 'small').write_text('hello sha-3\n')
size = 70 * 1024
(FROMDIR / 'sub' / 'big').write_bytes(Random(0).getrandbits(size * 8).to_bytes(size, 'big'))

src, dst = f'{FROMDIR}/', f'{TODIR}/'


def check_tree(label):
names = sorted(p.relative_to(FROMDIR) for p in walk_files(FROMDIR))
if names != sorted(p.relative_to(TODIR) for p in walk_files(TODIR)):
test_fail(f'{label}: file lists differ')
for rel in names:
assert_same(TODIR / rel, FROMDIR / rel, label=f'{label} {rel}')


def check_default_checksum():
rmtree(TODIR)
result = run_rsync('-a', '--debug=NSTR2', src, dst, capture_output=True, check=False)
if result.returncode:
test_fail(f'default checksum transfer failed: {result}')
lines = result.stdout.splitlines()
if legacy_checksum:
if 'Client checksum: md4' not in lines:
test_fail(f'protocol {protocol}: expected the MD4 default: {result.stdout!r}')
else:
for peer in ('Client', 'Server'):
prefix = f'{peer} checksum list (on client): '
offers = [line[len(prefix):].split() for line in lines if line.startswith(prefix)]
if len(offers) != 1 or set(offers[0]).intersection(ALGOS):
test_fail(f'{peer}: invalid default checksum offer: {offers!r}')
check_tree('auto')


def transfer_with_checksum_list(choices):
return subprocess.run(rsync_argv('-a', '--debug=NSTR2', src, dst),
env={**os.environ, 'RSYNC_CHECKSUM_LIST': choices},
capture_output=True, text=True)


def check_checksum_list(algo):
rmtree(TODIR)
result = transfer_with_checksum_list(algo)
expected = f'Client negotiated checksum: {algo}'
if legacy_checksum:
# Without negotiation the list may allow MD4 or refuse it, not replace it.
if result.returncode != 4 or 'Failed to negotiate a checksum choice.' not in result.stderr:
test_fail(f'protocol {protocol}: expected a SHA-3-only list to be refused: {result}')
rmtree(TODIR)
result = transfer_with_checksum_list(f'{algo} md4')
expected = 'Client checksum: md4'
if result.returncode or expected not in result.stdout.splitlines():
test_fail(f'RSYNC_CHECKSUM_LIST: expected {expected!r}: {result}')
check_tree('checksum list')


check_default_checksum()


for algo in ALGOS:
# Whole-file path (transfer checksum only).
rmtree(TODIR)
if algo not in available:
result = run_rsync('-a', f'--checksum-choice={algo}', src, dst,
capture_output=True, check=False)
if result.returncode != 4 or f'unknown checksum name: {algo}' not in result.stderr:
test_fail(f'{algo}: an unavailable digest was not refused: {result.returncode}: {result.stderr}')
continue
result = run_rsync('-a', f'--checksum-choice={algo}', '--debug=NSTR2', src, dst,
capture_output=True, check=False)
if result.returncode or f'Client checksum: {algo}' not in result.stdout.splitlines():
test_fail(f'explicit choice did not select {algo}: {result}')
check_tree(algo)
# Pre-transfer + transfer checksum path (-c forces file_checksum to run).
rmtree(TODIR)
run_rsync('-ac', f'--checksum-choice={algo}', src, dst)
check_tree(f'{algo} -c')
# --checksum-choice with an explicit xfer,file pair.
rmtree(TODIR)
run_rsync('-ac', f'--checksum-choice={algo},{algo}', src, dst)
check_tree(f'{algo},{algo} -c')

if available:
algo = available[0]
check_checksum_list(algo)
with (TODIR / 'sub' / 'big').open('r+b') as stream:
stream.seek(32 * 1024)
stream.write(b'modified locally\n')
result = run_rsync('-ac', '--no-whole-file', '--stats', f'--checksum-choice={algo}',
src, dst, capture_output=True)
for field in ('Matched', 'Literal'):
count = re.search(rf'^{field} data: ([0-9,]+) bytes$', result.stdout, re.MULTILINE)
if not count or int(count[1].replace(',', '')) == 0:
test_fail(f'{algo} delta: no {field.lower()} data: {result.stdout!r}')
check_tree(f'{algo} delta')