operations: normalize paths in SUM files

Paths like "./dir/file", as from find, and "/dir/file" were not matched,
and on Windows backslash was not accepted as path separator, as written
by Windows tools.

backend/hasher/commands.go

@@ -145,7 +145,7 @@ func (f *Fs) dbImport(ctx context.Context, hashName, sumRemote string, sticky bo
 	if err != nil {
 		return fmt.Errorf("cannot open sum file: %w", err)
 	}
-	hashes, err := operations.ParseSumFile(ctx, sumObj)
+	hashes, err := operations.ParseSumFile(ctx, sumObj, hashType)
 	if err != nil {
 		return fmt.Errorf("failed to parse sum file: %w", err)
 	}

cmd/checksum/checksum.go

@@ -37,6 +37,10 @@ If you supply the |--download| flag, it will download the data from the remote
 and calculate the content hash on the fly.  This can be useful for remotes
 that don't support hashes or if you really want to check all the data.
 
+The SUM file can be in the format of the standard md5sum/sha1sum tools.
+Lines starting with |#| or |;| are ignored. The file may be UTF-8, or
+UTF-16 with a byte order mark.
+
 Note that hash values in the SUM file are treated as case insensitive.
 `, "|", "`") + check.FlagsHelp,
 	Annotations: map[string]string{

fs/operations/check.go

@@ -8,6 +8,8 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"sync"
@@ -20,6 +22,8 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/march"
 	"github.com/rclone/rclone/lib/readers"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 	"golang.org/x/text/unicode/norm"
 )
 
@@ -430,7 +434,7 @@ func CheckSum(ctx context.Context, fsrc, fsum fs.Fs, sumFile string, hashType ha
 	if err != nil {
 		return fmt.Errorf("cannot open sum file: %w", err)
 	}
-	hashes, err := ParseSumFile(ctx, sumObj)
+	hashes, err := ParseSumFile(ctx, sumObj, hashType)
 	if err != nil {
 		return fmt.Errorf("failed to parse sum file: %w", err)
 	}
@@ -566,33 +570,41 @@ func (c *checkMarch) matchSum(ctx context.Context, sumHash, objHash string, obj
 type HashSums map[string]string
 
 // ParseSumFile parses a hash SUM file and returns hashes as a map
-func ParseSumFile(ctx context.Context, sumFile fs.Object) (HashSums, error) {
+func ParseSumFile(ctx context.Context, sumFile fs.Object, hashType hash.Type) (HashSums, error) {
 	rd, err := Open(ctx, sumFile)
 	if err != nil {
 		return nil, err
 	}
-	parser := bufio.NewReader(rd)
+	parser := bufio.NewReader(transform.NewReader(rd, unicode.BOMOverride(transform.Nop)))
 
 	const maxWarn = 3
 	numWarn := 0
 
-	re := regexp.MustCompile(`^([^ ]+) [ *](.+)$`)
+	re := regexp.MustCompile(`^([^ ]+) (?:\?([^ *]+))?[ *](.+)$`)
 	hashes := HashSums{}
+	marked := map[string]bool{} // files with a hash from a line marked with hashType
 	for lineNo := 0; true; lineNo++ {
-		lineBytes, _, err := parser.ReadLine()
-		if err == io.EOF {
+		line, err := parser.ReadString('\n')
+		if err == io.EOF && line == "" {
 			break
 		}
-		if err != nil {
+		if err != nil && err != io.EOF {
 			return nil, err
 		}
-		line := string(lineBytes)
-		if line == "" {
+		line = strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r")
+		if line == "" || line[0] == '#' || line[0] == ';' {
 			continue
 		}
+		escaped := line[0] == '\\'
+		if escaped {
+			line = line[1:]
+		}
 
 		fields := re.FindStringSubmatch(ApplyTransforms(ctx, line))
-		if fields == nil {
+		if fields != nil && escaped {
+			fields[3] = unescapeSumFileName(fields[3])
+		}
+		if fields == nil || fields[3] == "" {
 			numWarn++
 			if numWarn <= maxWarn {
 				fs.LogfCtx(ctx, sumFile, "improperly formatted checksum line %d", lineNo)
@@ -600,14 +612,30 @@ func ParseSumFile(ctx context.Context, sumFile fs.Object) (HashSums, error) {
 			continue
 		}
 
-		sum, file := fields[1], fields[2]
-		if hashes[file] != "" {
+		sum, algo, file := fields[1], fields[2], fields[3]
+		// Treat the path like a local path, where backslash is a separator
+		// on Windows, unless the escaping says it is a literal backslash
+		if !escaped {
+			file = filepath.ToSlash(file)
+		}
+		file = strings.TrimPrefix(path.Clean(file), "/")
+		if algo != "" && !strings.EqualFold(algo, hashType.String()) {
+			continue
+		}
+		// A marked line replaces an unmarked one, but not the reverse
+		if algo == "" && marked[file] {
+			continue
+		}
+		if hashes[file] != "" && (algo == "" || marked[file]) {
 			numWarn++
 			if numWarn <= maxWarn {
 				fs.LogfCtx(ctx, sumFile, "duplicate file on checksum line %d", lineNo)
 			}
 			continue
 		}
+		if algo != "" {
+			marked[file] = true
+		}
 
 		// We've standardised on lower case checksums in rclone internals.
 		hashes[file] = strings.ToLower(sum)
@@ -621,3 +649,32 @@ func ParseSumFile(ctx context.Context, sumFile fs.Object) (HashSums, error) {
 	}
 	return hashes, nil
 }
+
+// unescapeSumFileName reverses the escaping GNU md5sum and friends use for
+// file names containing backslash, newline or carriage return. It returns
+// "" if s is not validly escaped.
+func unescapeSumFileName(s string) string {
+	var b strings.Builder
+	b.Grow(len(s))
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if c == '\\' {
+			i++
+			if i == len(s) {
+				return ""
+			}
+			switch s[i] {
+			case '\\':
+				c = '\\'
+			case 'n':
+				c = '\n'
+			case 'r':
+				c = '\r'
+			default:
+				return ""
+			}
+		}
+		b.WriteByte(c)
+	}
+	return b.String()
+}

fs/operations/check_test.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"runtime"
 	"sort"
 	"strings"
 	"sync"
@@ -23,6 +24,8 @@ import (
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/unicode"
 	"golang.org/x/text/unicode/norm"
 )
 
@@ -297,6 +300,28 @@ func TestParseSumFile(t *testing.T) {
 		{"7", " \t", " file7 ", false},
 		{"", "  ", "file8", false},
 		{"", "", "file9", false},
+		{"10", "  ", "ファイル10", true},
+		{"#", "  ", "comment11", false},
+		{";", " *", "comment12", false},
+		{"13", " ?SHA1*", "file13", true},
+		{"14", " ?sha1 ", "file14", true},
+		{"15", " ?MD5*", "file15", false},
+		{"16", " *", "file16", false},
+		{"17", " ?SHA1*", "file16", true},
+		{"18", " ?SHA1*", "file18", true},
+		{"19", " *", "file18", false},
+		{"20", " ?*", "file20", false},
+		{"21", "  ", strings.Repeat("dir/", 1100) + "file21", true},
+	}
+
+	encodings := []struct {
+		name string
+		enc  encoding.Encoding
+	}{
+		{"UTF-8", encoding.Nop},
+		{"UTF-8 with BOM", unicode.UTF8BOM},
+		{"UTF-16LE with BOM", unicode.UTF16(unicode.LittleEndian, unicode.UseBOM)},
+		{"UTF-16BE with BOM", unicode.UTF16(unicode.BigEndian, unicode.UseBOM)},
 	}
 
 	for _, eol := range []string{"\n", "\r\n"} {
@@ -309,18 +334,60 @@ func TestParseSumFile(t *testing.T) {
 			}
 		}
 
-		_ = r.WriteObject(ctx, sumFile, data.String(), t1)
-		file := fstest.NewObject(ctx, t, r.Fremote, sumFile)
-		sums, err := operations.ParseSumFile(ctx, file)
-		assert.NoError(t, err)
+		for _, e := range encodings {
+			content, err := e.enc.NewEncoder().String(data.String())
+			require.NoError(t, err)
 
-		assert.Equal(t, wantNum, len(sums))
-		for _, s := range samples {
-			if s.ok {
-				assert.Equal(t, s.hash, sums[s.name])
+			_ = r.WriteObject(ctx, sumFile, content, t1)
+			file := fstest.NewObject(ctx, t, r.Fremote, sumFile)
+			sums, err := operations.ParseSumFile(ctx, file, hash.SHA1)
+			assert.NoError(t, err)
+
+			assert.Equal(t, wantNum, len(sums), "%s %q", e.name, eol)
+			for _, s := range samples {
+				if s.ok {
+					assert.Equal(t, s.hash, sums[s.name], "%s %q", e.name, eol)
+				}
 			}
 		}
 	}
+
+	// The last line doesn't need a line ending
+	_ = r.WriteObject(ctx, sumFile, "1  file1\n2  file2", t1)
+	file := fstest.NewObject(ctx, t, r.Fremote, sumFile)
+	sums, err := operations.ParseSumFile(ctx, file, hash.SHA1)
+	require.NoError(t, err)
+	assert.Equal(t, operations.HashSums{"file1": "1", "file2": "2"}, sums)
+
+	// A leading backslash marks an escaped file name, as written by GNU md5sum
+	_ = r.WriteObject(ctx, sumFile, strings.Join([]string{
+		`\1  dir\\file1`,
+		`\2 *a\nb\rc`,
+		`\4  invalid\x`,
+		`\5  invalid\`,
+	}, "\n"), t1)
+	file = fstest.NewObject(ctx, t, r.Fremote, sumFile)
+	sums, err = operations.ParseSumFile(ctx, file, hash.SHA1)
+	require.NoError(t, err)
+	assert.Equal(t, operations.HashSums{`dir\file1`: "1", "a\nb\rc": "2"}, sums)
+
+	// Paths are cleaned, and an unescaped backslash is a path separator on Windows
+	_ = r.WriteObject(ctx, sumFile, strings.Join([]string{
+		`1  ./file1`,
+		`2  /dir/file2`,
+		`3  dir//file3`,
+		`4 *dir\file4`,
+		`\5  dir\\file5`,
+	}, "\n"), t1)
+	file = fstest.NewObject(ctx, t, r.Fremote, sumFile)
+	sums, err = operations.ParseSumFile(ctx, file, hash.SHA1)
+	require.NoError(t, err)
+	want := operations.HashSums{"file1": "1", "dir/file2": "2", "dir/file3": "3", `dir\file4`: "4", `dir\file5`: "5"}
+	if runtime.GOOS == "windows" {
+		delete(want, `dir\file4`)
+		want["dir/file4"] = "4"
+	}
+	assert.Equal(t, want, sums)
 }
 
 func testCheckSum(t *testing.T, download bool) {