mega: fix key decoding - FIXME VENDOR PATCH DO NOT MERGE

See: https://forum.rclone.org/t/problem-to-login-with-mega/12276

.github/workflows/build.yml

@@ -104,7 +104,8 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v1
         with:
-          path: ./src/github.com/${{ github.repository }}
+          # Checkout into a fixed path to avoid import path problems on go < 1.11
+          path: ./src/github.com/rclone/rclone
 
       - name: Install Go
         uses: actions/setup-go@v1
@@ -201,7 +202,8 @@ jobs:
         env:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
         # working-directory: '$(modulePath)'
-        if: matrix.deploy && github.head_ref == ''
+        # Deploy binaries if enabled in config && not a PR && not a fork
+        if: matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
 
   xgo:
     timeout-minutes: 60
@@ -213,7 +215,8 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v1
         with:
-          path: ./src/github.com/${{ github.repository }}
+          # Checkout into a fixed path to avoid import path problems on go < 1.11
+          path: ./src/github.com/rclone/rclone
 
       - name: Set environment variables
         shell: bash
@@ -247,4 +250,5 @@ jobs:
           make circleci_upload
         env:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
-        if: github.head_ref == ''
+        # Upload artifacts if not a PR && not a fork
+        if: github.head_ref == '' && github.repository == 'rclone/rclone'

CONTRIBUTING.md

@@ -82,13 +82,9 @@ You patch will get reviewed and you might get asked to fix some stuff.
 If so, then make the changes in the same branch, squash the commits,
 rebase it to master then push it to GitHub with `--force`.
 
-## Enabling CI for your fork ##
+## CI for your fork ##
 
-The CI config files for rclone have taken care of forks of the project, so you can enable CI for your fork repo easily.
-
-rclone currently uses [Travis CI](https://travis-ci.org/), [AppVeyor](https://ci.appveyor.com/), and
-[Circle CI](https://circleci.com/) to build the project. To enable them for your fork, simply go into their
-websites, find your fork of rclone, and enable building there.
+rclone currently uses [GitHub Actions](https://github.com/rclone/rclone/actions) to build and test the project, which should be automatically available for your fork too from the `Actions` tab in your repository.
 
 ## Testing ##
 

README.md

@@ -34,6 +34,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
   * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
+  * GetSky [:page_facing_up:](https://rclone.org/jottacloud/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)
   * Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)

backend/cache/cache_mount_other_test.go

@@ -0,0 +1,20 @@
+// +build !linux !go1.11
+// +build !darwin !go1.11
+// +build !freebsd !go1.11
+// +build !windows
+
+package cache_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+)
+
+func (r *run) mountFs(t *testing.T, f fs.Fs) {
+	panic("mountFs not defined for this platform")
+}
+
+func (r *run) unmountFs(t *testing.T, f fs.Fs) {
+	panic("unmountFs not defined for this platform")
+}

backend/cache/cache_mount_unix_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,!windows
+// +build linux,go1.11 darwin,go1.11 freebsd,go1.11
 
 package cache_test
 

backend/drive/drive.go

@@ -1546,15 +1546,23 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in <-chan list
 		listRSlices{dirs, paths}.Sort()
 		var iErr error
 		_, err := f.list(ctx, dirs, "", false, false, false, func(item *drive.File) bool {
-			// shared with me items have no parents when at the root
-			if f.opt.SharedWithMe && len(item.Parents) == 0 && len(paths) == 1 && paths[0] == "" {
-				item.Parents = dirs
-			}
 			for _, parent := range item.Parents {
-				// only handle parents that are in the requested dirs list
-				i := sort.SearchStrings(dirs, parent)
-				if i == len(dirs) || dirs[i] != parent {
-					continue
+				var i int
+				// If only one item in paths then no need to search for the ID
+				// assuming google drive is doing its job properly.
+				//
+				// Note that we at the root when len(paths) == 1 && paths[0] == ""
+				if len(paths) == 1 {
+					// don't check parents at root because
+					// - shared with me items have no parents at the root
+					// - if using a root alias, eg "root" or "appDataFolder" the ID won't match
+					i = 0
+				} else {
+					// only handle parents that are in the requested dirs list if not at root
+					i = sort.SearchStrings(dirs, parent)
+					if i == len(dirs) || dirs[i] != parent {
+						continue
+					}
 				}
 				remote := path.Join(paths[i], item.Name)
 				entry, err := f.itemToDirEntry(remote, item)

backend/jottacloud/api/types.go

@@ -54,6 +54,37 @@ type LoginToken struct {
 	AuthToken     string `json:"auth_token"`
 }
 
+// WellKnown contains some configuration parameters for setting up endpoints
+type WellKnown struct {
+	Issuer                                     string   `json:"issuer"`
+	AuthorizationEndpoint                      string   `json:"authorization_endpoint"`
+	TokenEndpoint                              string   `json:"token_endpoint"`
+	TokenIntrospectionEndpoint                 string   `json:"token_introspection_endpoint"`
+	UserinfoEndpoint                           string   `json:"userinfo_endpoint"`
+	EndSessionEndpoint                         string   `json:"end_session_endpoint"`
+	JwksURI                                    string   `json:"jwks_uri"`
+	CheckSessionIframe                         string   `json:"check_session_iframe"`
+	GrantTypesSupported                        []string `json:"grant_types_supported"`
+	ResponseTypesSupported                     []string `json:"response_types_supported"`
+	SubjectTypesSupported                      []string `json:"subject_types_supported"`
+	IDTokenSigningAlgValuesSupported           []string `json:"id_token_signing_alg_values_supported"`
+	UserinfoSigningAlgValuesSupported          []string `json:"userinfo_signing_alg_values_supported"`
+	RequestObjectSigningAlgValuesSupported     []string `json:"request_object_signing_alg_values_supported"`
+	ResponseNodesSupported                     []string `json:"response_modes_supported"`
+	RegistrationEndpoint                       string   `json:"registration_endpoint"`
+	TokenEndpointAuthMethodsSupported          []string `json:"token_endpoint_auth_methods_supported"`
+	TokenEndpointAuthSigningAlgValuesSupported []string `json:"token_endpoint_auth_signing_alg_values_supported"`
+	ClaimsSupported                            []string `json:"claims_supported"`
+	ClaimTypesSupported                        []string `json:"claim_types_supported"`
+	ClaimsParameterSupported                   bool     `json:"claims_parameter_supported"`
+	ScopesSupported                            []string `json:"scopes_supported"`
+	RequestParameterSupported                  bool     `json:"request_parameter_supported"`
+	RequestURIParameterSupported               bool     `json:"request_uri_parameter_supported"`
+	CodeChallengeMethodsSupported              []string `json:"code_challenge_methods_supported"`
+	TLSClientCertificateBoundAccessTokens      bool     `json:"tls_client_certificate_bound_access_tokens"`
+	IntrospectionEndpoint                      string   `json:"introspection_endpoint"`
+}
+
 // TokenJSON is the struct representing the HTTP response from OAuth2
 // providers returning a token in JSON form.
 type TokenJSON struct {

backend/jottacloud/jottacloud.go

@@ -49,10 +49,11 @@ const (
 	rootURL           = "https://www.jottacloud.com/jfs/"
 	apiURL            = "https://api.jottacloud.com/"
 	baseURL           = "https://www.jottacloud.com/"
-	tokenURL          = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
+	defaultTokenURL   = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
 	cachePrefix       = "rclone-jcmd5-"
 	configDevice      = "device"
 	configMountpoint  = "mountpoint"
+	configTokenURL    = "tokenURL"
 	configVersion     = 1
 )
 
@@ -61,8 +62,8 @@ var (
 	oauthConfig = &oauth2.Config{
 		ClientID: "jottacli",
 		Endpoint: oauth2.Endpoint{
-			AuthURL:  tokenURL,
-			TokenURL: tokenURL,
+			AuthURL:  defaultTokenURL,
+			TokenURL: defaultTokenURL,
 		},
 		RedirectURL: oauthutil.RedirectLocalhostURL,
 	}
@@ -85,8 +86,6 @@ func init() {
 					log.Fatalf("Failed to parse config version - corrupted config")
 				}
 				refresh = ver != configVersion
-			} else {
-				refresh = true
 			}
 
 			if refresh {
@@ -109,7 +108,7 @@ func init() {
 			fmt.Printf("Login Token> ")
 			loginToken := config.ReadLine()
 
-			token, err := doAuth(ctx, srv, loginToken)
+			token, err := doAuth(ctx, srv, loginToken, m)
 			if err != nil {
 				log.Fatalf("Failed to get oauth token: %s", err)
 			}
@@ -244,12 +243,13 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 }
 
 // doAuth runs the actual token request
-func doAuth(ctx context.Context, srv *rest.Client, loginTokenBase64 string) (token oauth2.Token, err error) {
+func doAuth(ctx context.Context, srv *rest.Client, loginTokenBase64 string, m configmap.Mapper) (token oauth2.Token, err error) {
 	loginTokenBytes, err := base64.StdEncoding.DecodeString(loginTokenBase64)
 	if err != nil {
 		return token, err
 	}
 
+	// decode login token
 	var loginToken api.LoginToken
 	decoder := json.NewDecoder(bytes.NewReader(loginTokenBytes))
 	err = decoder.Decode(&loginToken)
@@ -257,17 +257,22 @@ func doAuth(ctx context.Context, srv *rest.Client, loginTokenBase64 string) (tok
 		return token, err
 	}
 
-	// we don't seem to need any data from this link but the API is not happy if skip it
+	// retrieve endpoint urls
 	opts := rest.Opts{
-		Method:     "GET",
-		RootURL:    loginToken.WellKnownLink,
-		NoResponse: true,
+		Method:  "GET",
+		RootURL: loginToken.WellKnownLink,
 	}
-	_, err = srv.Call(ctx, &opts)
+	var wellKnown api.WellKnown
+	_, err = srv.CallJSON(ctx, &opts, nil, &wellKnown)
 	if err != nil {
 		return token, err
 	}
 
+	// save the tokenurl
+	oauthConfig.Endpoint.AuthURL = wellKnown.TokenEndpoint
+	oauthConfig.Endpoint.TokenURL = wellKnown.TokenEndpoint
+	m.Set(configTokenURL, wellKnown.TokenEndpoint)
+
 	// prepare out token request with username and password
 	values := url.Values{}
 	values.Set("client_id", "jottacli")
@@ -459,6 +464,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		return nil, err
 	}
 
+	// Check config version
 	var ok bool
 	var version string
 	if version, ok = m.Get("configVersion"); ok {
@@ -472,15 +478,23 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		return nil, errors.New("Outdated config - please reconfigure this backend")
 	}
 
-	rootIsDir := strings.HasSuffix(root, "/")
-	root = parsePath(root)
+	// if custome endpoints are set use them else stick with defaults
+	if tokenURL, ok := m.Get(configTokenURL); ok {
+		oauthConfig.Endpoint.TokenURL = tokenURL
+		// jottacloud is weird. we need to use the tokenURL as authURL
+		oauthConfig.Endpoint.AuthURL = tokenURL
+	}
 
+	// Create OAuth Client
 	baseClient := fshttp.NewClient(fs.Config)
 	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, oauthConfig, baseClient)
 	if err != nil {
 		return nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
 	}
 
+	rootIsDir := strings.HasSuffix(root, "/")
+	root = parsePath(root)
+
 	f := &Fs{
 		name:   name,
 		root:   root,

backend/local/local.go

@@ -956,7 +956,17 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if !o.translatedLink {
 		f, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
 		if err != nil {
-			return err
+			if runtime.GOOS == "windows" && os.IsPermission(err) {
+				// If permission denied on Windows might be trying to update a
+				// hidden file, in which case try opening without CREATE
+				// See: https://stackoverflow.com/questions/13215716/ioerror-errno-13-permission-denied-when-trying-to-open-hidden-file-in-w-mod
+				f, err = file.OpenFile(o.path, os.O_WRONLY|os.O_TRUNC, 0666)
+				if err != nil {
+					return err
+				}
+			} else {
+				return err
+			}
 		}
 		// Pre-allocate the file for performance reasons
 		err = preAllocate(src.Size(), f)

backend/s3/s3.go

@@ -14,7 +14,9 @@ What happens if you CTRL-C a multipart upload
 */
 
 import (
+	"bytes"
 	"context"
+	"crypto/md5"
 	"encoding/base64"
 	"encoding/hex"
 	"encoding/xml"
@@ -24,6 +26,7 @@ import (
 	"net/url"
 	"path"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -34,12 +37,12 @@ import (
 	"github.com/aws/aws-sdk-go/aws/corehandlers"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
+	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/defaults"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/ncw/swift"
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
@@ -52,7 +55,9 @@ import (
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/sync/errgroup"
 )
 
 const enc = encodings.S3
@@ -160,6 +165,9 @@ func init() {
 			}, {
 				Value: "ap-south-1",
 				Help:  "Asia Pacific (Mumbai)\nNeeds location constraint ap-south-1.",
+			}, {
+				Value: "ap-east-1",
+				Help:  "Asia Patific (Hong Kong) Region\nNeeds location constraint ap-east-1.",
 			}, {
 				Value: "sa-east-1",
 				Help:  "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.",
@@ -428,6 +436,9 @@ func init() {
 			}, {
 				Value: "ap-south-1",
 				Help:  "Asia Pacific (Mumbai)",
+			}, {
+				Value: "ap-east-1",
+				Help:  "Asia Pacific (Hong Kong)",
 			}, {
 				Value: "sa-east-1",
 				Help:  "South America (Sao Paulo) Region.",
@@ -755,7 +766,9 @@ if false then rclone will use virtual path style. See [the AWS S3
 docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro)
 for more info.
 
-Some providers (eg Aliyun OSS or Netease COS) require this set to false.`,
+Some providers (eg AWS, Aliyun OSS or Netease COS) require this set to
+false - rclone will do this automatically based on the provider
+setting.`,
 			Default:  true,
 			Advanced: true,
 		}, {
@@ -787,6 +800,17 @@ WARNING: Storing parts of an incomplete multipart upload counts towards space us
 `,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "list_chunk",
+			Help: `Size of listing chunk (response list for each ListObject S3 request).
+
+This option is also known as "MaxKeys", "max-items", or "page-size" from the AWS S3 specification.
+Most services truncate the response list to 1000 objects even if requested more than that.
+In AWS S3 this is a global maximum and cannot be changed, see [AWS S3](https://docs.aws.amazon.com/cli/latest/reference/s3/ls.html).
+In Ceph, this can be increased with the "rgw list buckets max chunk" option.
+`,
+			Default:  1000,
+			Advanced: true,
 		}},
 	})
 }
@@ -795,10 +819,10 @@ WARNING: Storing parts of an incomplete multipart upload counts towards space us
 const (
 	metaMtime           = "Mtime"                // the meta key to store mtime in - eg X-Amz-Meta-Mtime
 	metaMD5Hash         = "Md5chksum"            // the meta key to store md5hash in
-	listChunkSize       = 1000                   // number of items to read at once
 	maxRetries          = 10                     // number of retries to make of operations
 	maxSizeForCopy      = 5 * 1024 * 1024 * 1024 // The maximum size of object we can COPY
-	minChunkSize        = fs.SizeSuffix(s3manager.MinUploadPartSize)
+	maxUploadParts      = 10000                  // maximum allowed number of parts in a multi-part upload
+	minChunkSize        = fs.SizeSuffix(1024 * 1024 * 5)
 	defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
 	maxUploadCutoff     = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
 	minSleep            = 10 * time.Millisecond // In case of error, start at 10ms sleep.
@@ -828,6 +852,7 @@ type Options struct {
 	V2Auth                bool          `config:"v2_auth"`
 	UseAccelerateEndpoint bool          `config:"use_accelerate_endpoint"`
 	LeavePartsOnError     bool          `config:"leave_parts_on_error"`
+	ListChunk             int64         `config:"list_chunk"`
 }
 
 // Fs represents a remote s3 server
@@ -985,6 +1010,11 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 			}),
 			ExpiryWindow: 3 * time.Minute,
 		},
+
+		// Pick up IAM role if we are in EKS
+		&stscreds.WebIdentityRoleProvider{
+			ExpiryWindow: 3 * time.Minute,
+		},
 	}
 	cred := credentials.NewChainCredentials(providers)
 
@@ -1006,7 +1036,7 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 	if opt.Region == "" {
 		opt.Region = "us-east-1"
 	}
-	if opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.UseAccelerateEndpoint {
+	if opt.Provider == "AWS" || opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.UseAccelerateEndpoint {
 		opt.ForcePathStyle = false
 	}
 	awsConfig := aws.NewConfig().
@@ -1254,7 +1284,6 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 	if directory != "" {
 		directory += "/"
 	}
-	maxKeys := int64(listChunkSize)
 	delimiter := ""
 	if !recurse {
 		delimiter = "/"
@@ -1282,7 +1311,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 			Bucket:    &bucket,
 			Delimiter: &delimiter,
 			Prefix:    &directory,
-			MaxKeys:   &maxKeys,
+			MaxKeys:   &f.opt.ListChunk,
 			Marker:    marker,
 		}
 		if urlEncodeListings {
@@ -2007,6 +2036,193 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 var warnStreamUpload sync.Once
 
+func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (err error) {
+	f := o.fs
+
+	// make concurrency machinery
+	concurrency := f.opt.UploadConcurrency
+	if concurrency < 1 {
+		concurrency = 1
+	}
+	bufs := make(chan []byte, concurrency)
+	defer func() {
+		// empty the channel on exit
+		close(bufs)
+		for range bufs {
+		}
+	}()
+	for i := 0; i < concurrency; i++ {
+		bufs <- nil
+	}
+
+	// calculate size of parts
+	partSize := int(f.opt.ChunkSize)
+
+	// size can be -1 here meaning we don't know the size of the incoming file.  We use ChunkSize
+	// buffers here (default 5MB). With a maximum number of parts (10,000) this will be a file of
+	// 48GB which seems like a not too unreasonable limit.
+	if size == -1 {
+		warnStreamUpload.Do(func() {
+			fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
+				f.opt.ChunkSize, fs.SizeSuffix(partSize*maxUploadParts))
+		})
+	} else {
+		// Adjust partSize until the number of parts is small enough.
+		if size/int64(partSize) >= maxUploadParts {
+			// Calculate partition size rounded up to the nearest MB
+			partSize = int((((size / maxUploadParts) >> 20) + 1) << 20)
+		}
+	}
+
+	var cout *s3.CreateMultipartUploadOutput
+	err = f.pacer.Call(func() (bool, error) {
+		var err error
+		cout, err = f.c.CreateMultipartUploadWithContext(ctx, &s3.CreateMultipartUploadInput{
+			Bucket:               req.Bucket,
+			ACL:                  req.ACL,
+			Key:                  req.Key,
+			ContentType:          req.ContentType,
+			Metadata:             req.Metadata,
+			ServerSideEncryption: req.ServerSideEncryption,
+			SSEKMSKeyId:          req.SSEKMSKeyId,
+			StorageClass:         req.StorageClass,
+		})
+		return f.shouldRetry(err)
+	})
+	if err != nil {
+		return errors.Wrap(err, "multipart upload failed to initialise")
+	}
+	uid := cout.UploadId
+
+	defer func() {
+		if o.fs.opt.LeavePartsOnError {
+			return
+		}
+		if err != nil {
+			// We can try to abort the upload, but ignore the error.
+			fs.Debugf(o, "Cancelling multipart upload")
+			errCancel := f.pacer.Call(func() (bool, error) {
+				_, err := f.c.AbortMultipartUploadWithContext(ctx, &s3.AbortMultipartUploadInput{
+					Bucket:       req.Bucket,
+					Key:          req.Key,
+					UploadId:     uid,
+					RequestPayer: req.RequestPayer,
+				})
+				return f.shouldRetry(err)
+			})
+			if errCancel != nil {
+				fs.Debugf(o, "Failed to cancel multipart upload: %v", errCancel)
+			}
+		}
+	}()
+
+	var (
+		g, gCtx  = errgroup.WithContext(ctx)
+		finished = false
+		partsMu  sync.Mutex // to protect parts
+		parts    []*s3.CompletedPart
+		off      int64
+	)
+
+	for partNum := int64(1); !finished; partNum++ {
+		// Get a block of memory from the channel (which limits concurrency)
+		buf := <-bufs
+		if buf == nil {
+			buf = make([]byte, partSize)
+		}
+
+		// Read the chunk
+		var n int
+		n, err = readers.ReadFill(in, buf) // this can never return 0, nil
+		if err == io.EOF {
+			if n == 0 && partNum != 1 { // end if no data and if not first chunk
+				break
+			}
+			finished = true
+		} else if err != nil {
+			return errors.Wrap(err, "multipart upload failed to read source")
+		}
+		buf = buf[:n]
+
+		partNum := partNum
+		fs.Debugf(o, "multipart upload starting chunk %d size %v offset %v/%v", partNum, fs.SizeSuffix(n), fs.SizeSuffix(off), fs.SizeSuffix(size))
+		off += int64(n)
+		g.Go(func() (err error) {
+			partLength := int64(len(buf))
+
+			// create checksum of buffer for integrity checking
+			md5sumBinary := md5.Sum(buf)
+			md5sum := base64.StdEncoding.EncodeToString(md5sumBinary[:])
+
+			err = f.pacer.Call(func() (bool, error) {
+				uploadPartReq := &s3.UploadPartInput{
+					Body:                 bytes.NewReader(buf),
+					Bucket:               req.Bucket,
+					Key:                  req.Key,
+					PartNumber:           &partNum,
+					UploadId:             uid,
+					ContentMD5:           &md5sum,
+					ContentLength:        &partLength,
+					RequestPayer:         req.RequestPayer,
+					SSECustomerAlgorithm: req.SSECustomerAlgorithm,
+					SSECustomerKey:       req.SSECustomerKey,
+					SSECustomerKeyMD5:    req.SSECustomerKeyMD5,
+				}
+				uout, err := f.c.UploadPartWithContext(gCtx, uploadPartReq)
+				if err != nil {
+					if partNum <= int64(concurrency) {
+						return f.shouldRetry(err)
+					}
+					// retry all chunks once have done the first batch
+					return true, err
+				}
+				partsMu.Lock()
+				parts = append(parts, &s3.CompletedPart{
+					PartNumber: &partNum,
+					ETag:       uout.ETag,
+				})
+				partsMu.Unlock()
+
+				return false, nil
+			})
+
+			// return the memory
+			bufs <- buf[:partSize]
+
+			if err != nil {
+				return errors.Wrap(err, "multipart upload failed to upload part")
+			}
+			return nil
+		})
+	}
+	err = g.Wait()
+	if err != nil {
+		return err
+	}
+
+	// sort the completed parts by part number
+	sort.Slice(parts, func(i, j int) bool {
+		return *parts[i].PartNumber < *parts[j].PartNumber
+	})
+
+	err = f.pacer.Call(func() (bool, error) {
+		_, err := f.c.CompleteMultipartUploadWithContext(ctx, &s3.CompleteMultipartUploadInput{
+			Bucket: req.Bucket,
+			Key:    req.Key,
+			MultipartUpload: &s3.CompletedMultipartUpload{
+				Parts: parts,
+			},
+			RequestPayer: req.RequestPayer,
+			UploadId:     uid,
+		})
+		return f.shouldRetry(err)
+	})
+	if err != nil {
+		return errors.Wrap(err, "multipart upload failed to finalise")
+	}
+	return nil
+}
+
 // Update the Object from in with modTime and size
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	bucket, bucketPath := o.split()
@@ -2018,41 +2234,19 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	size := src.Size()
 
 	multipart := size < 0 || size >= int64(o.fs.opt.UploadCutoff)
-	var uploader *s3manager.Uploader
-	if multipart {
-		uploader = s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) {
-			u.Concurrency = o.fs.opt.UploadConcurrency
-			u.LeavePartsOnError = o.fs.opt.LeavePartsOnError
-			u.S3 = o.fs.c
-			u.PartSize = int64(o.fs.opt.ChunkSize)
-
-			// size can be -1 here meaning we don't know the size of the incoming file.  We use ChunkSize
-			// buffers here (default 5MB). With a maximum number of parts (10,000) this will be a file of
-			// 48GB which seems like a not too unreasonable limit.
-			if size == -1 {
-				warnStreamUpload.Do(func() {
-					fs.Logf(o.fs, "Streaming uploads using chunk size %v will have maximum file size of %v",
-						o.fs.opt.ChunkSize, fs.SizeSuffix(u.PartSize*s3manager.MaxUploadParts))
-				})
-				return
-			}
-			// Adjust PartSize until the number of parts is small enough.
-			if size/u.PartSize >= s3manager.MaxUploadParts {
-				// Calculate partition size rounded up to the nearest MB
-				u.PartSize = (((size / s3manager.MaxUploadParts) >> 20) + 1) << 20
-			}
-		})
-	}
 
 	// Set the mtime in the meta data
 	metadata := map[string]*string{
 		metaMtime: aws.String(swift.TimeToFloatString(modTime)),
 	}
 
-	// read the md5sum if available for non multpart and if
-	// disable checksum isn't present.
+	// read the md5sum if available
+	// - for non multpart
+	//    - so we can add a ContentMD5
+	// - for multipart provided checksums aren't disabled
+	//    - so we can add the md5sum in the metadata as metaMD5Hash
 	var md5sum string
-	if !multipart && !o.fs.opt.DisableChecksum {
+	if !multipart || !o.fs.opt.DisableChecksum {
 		hash, err := src.Hash(ctx, hash.MD5)
 		if err == nil && matchMd5.MatchString(hash) {
 			hashBytes, err := hex.DecodeString(hash)
@@ -2067,52 +2261,32 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Guess the content type
 	mimeType := fs.MimeType(ctx, src)
+	req := s3.PutObjectInput{
+		Bucket:      &bucket,
+		ACL:         &o.fs.opt.ACL,
+		Key:         &bucketPath,
+		ContentType: &mimeType,
+		Metadata:    metadata,
+	}
+	if md5sum != "" {
+		req.ContentMD5 = &md5sum
+	}
+	if o.fs.opt.ServerSideEncryption != "" {
+		req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
+	}
+	if o.fs.opt.SSEKMSKeyID != "" {
+		req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
+	}
+	if o.fs.opt.StorageClass != "" {
+		req.StorageClass = &o.fs.opt.StorageClass
+	}
+
 	if multipart {
-		req := s3manager.UploadInput{
-			Bucket:      &bucket,
-			ACL:         &o.fs.opt.ACL,
-			Key:         &bucketPath,
-			Body:        in,
-			ContentType: &mimeType,
-			Metadata:    metadata,
-			//ContentLength: &size,
-		}
-		if o.fs.opt.ServerSideEncryption != "" {
-			req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
-		}
-		if o.fs.opt.SSEKMSKeyID != "" {
-			req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
-		}
-		if o.fs.opt.StorageClass != "" {
-			req.StorageClass = &o.fs.opt.StorageClass
-		}
-		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			_, err = uploader.UploadWithContext(ctx, &req)
-			return o.fs.shouldRetry(err)
-		})
+		err = o.uploadMultipart(ctx, &req, size, in)
 		if err != nil {
 			return err
 		}
 	} else {
-		req := s3.PutObjectInput{
-			Bucket:      &bucket,
-			ACL:         &o.fs.opt.ACL,
-			Key:         &bucketPath,
-			ContentType: &mimeType,
-			Metadata:    metadata,
-		}
-		if md5sum != "" {
-			req.ContentMD5 = &md5sum
-		}
-		if o.fs.opt.ServerSideEncryption != "" {
-			req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
-		}
-		if o.fs.opt.SSEKMSKeyID != "" {
-			req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
-		}
-		if o.fs.opt.StorageClass != "" {
-			req.StorageClass = &o.fs.opt.StorageClass
-		}
 
 		// Create the request
 		putObj, _ := o.fs.c.PutObjectRequest(&req)

bin/win-build.bat

@@ -1,5 +1,6 @@
 @echo off
 echo Setting environment variables for mingw+WinFsp compile
-set GOPATH=X:\go
-set PATH=C:\Program Files\mingw-w64\i686-7.1.0-win32-dwarf-rt_v5-rev0\mingw32\bin;%PATH%
+set GOPATH=Z:\go
+rem set PATH=C:\Program Files\mingw-w64\i686-7.1.0-win32-dwarf-rt_v5-rev0\mingw32\bin;%PATH%
+set PATH=C:\Program Files\mingw-w64\x86_64-8.1.0-win32-seh-rt_v6-rev0\mingw64\bin;%PATH%
 set CPATH=C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse

cmd/help.go

@@ -10,6 +10,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configflags"
 	"github.com/rclone/rclone/fs/filter/filterflags"
+	"github.com/rclone/rclone/fs/log/logflags"
 	"github.com/rclone/rclone/fs/rc/rcflags"
 	"github.com/rclone/rclone/lib/atexit"
 	"github.com/spf13/cobra"
@@ -169,6 +170,7 @@ func setupRootCommand(rootCmd *cobra.Command) {
 	configflags.AddFlags(pflag.CommandLine)
 	filterflags.AddFlags(pflag.CommandLine)
 	rcflags.AddFlags(pflag.CommandLine)
+	logflags.AddFlags(pflag.CommandLine)
 
 	Root.Run = runRoot
 	Root.Flags().BoolVarP(&version, "version", "V", false, "Print the version number")

cmd/mount/dir.go

@@ -1,4 +1,4 @@
-// +build linux darwin freebsd
+// +build linux,go1.11 darwin,go1.11 freebsd,go1.11
 
 package mount
 

cmd/mount/file.go

@@ -1,4 +1,4 @@
-// +build linux darwin freebsd
+// +build linux,go1.11 darwin,go1.11 freebsd,go1.11
 
 package mount
 

cmd/mount/fs.go

@@ -1,6 +1,6 @@
 // FUSE main Fs
 
-// +build linux darwin freebsd
+// +build linux,go1.11 darwin,go1.11 freebsd,go1.11
 
 package mount
 

cmd/mount/handle.go

@@ -1,4 +1,4 @@
-// +build linux darwin freebsd
+// +build linux,go1.11 darwin,go1.11 freebsd,go1.11
 
 package mount
 

cmd/mount/mount.go

@@ -1,6 +1,6 @@
 // Package mount implents a FUSE mounting system for rclone remotes.
 
-// +build linux darwin freebsd
+// +build linux,go1.11 darwin,go1.11 freebsd,go1.11
 
 package mount
 

cmd/mount/mount_test.go

@@ -1,4 +1,4 @@
-// +build linux darwin freebsd
+// +build linux,go1.11 darwin,go1.11 freebsd,go1.11
 
 package mount
 

cmd/mount/mount_unsupported.go

@@ -1,6 +1,14 @@
 // Build for mount for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build !linux,!darwin,!freebsd
+// Invert the build constraint: linux,go1.11 darwin,go1.11 freebsd,go1.11
+//
+// !((linux&&go1.11) || (darwin&&go1.11) || (freebsd&&go1.11))
+// == !(linux&&go1.11) && !(darwin&&go1.11) && !(freebsd&&go1.11))
+// == (!linux || !go1.11) && (!darwin || go1.11) && (!freebsd || !go1.11))
+
+// +build !linux !go1.11
+// +build !darwin !go1.11
+// +build !freebsd !go1.11
 
 package mount

cmd/mountlib/mounttest/fs.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"os"
@@ -22,6 +23,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/lib/file"
 	"github.com/rclone/rclone/vfs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -345,9 +347,36 @@ func (r *Run) waitForWriters() {
 	run.vfs.WaitForWriters(10 * time.Second)
 }
 
+// writeFile writes data to a file named by filename.
+// If the file does not exist, WriteFile creates it with permissions perm;
+// otherwise writeFile truncates it before writing.
+// If there is an error writing then writeFile
+// deletes it an existing file and tries again.
+func writeFile(filename string, data []byte, perm os.FileMode) error {
+	f, err := file.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm)
+	if err != nil {
+		err = os.Remove(filename)
+		if err != nil {
+			return err
+		}
+		f, err = file.OpenFile(filename, os.O_WRONLY|os.O_CREATE, perm)
+		if err != nil {
+			return err
+		}
+	}
+	n, err := f.Write(data)
+	if err == nil && n < len(data) {
+		err = io.ErrShortWrite
+	}
+	if err1 := f.Close(); err == nil {
+		err = err1
+	}
+	return err
+}
+
 func (r *Run) createFile(t *testing.T, filepath string, contents string) {
 	filepath = r.path(filepath)
-	err := ioutil.WriteFile(filepath, []byte(contents), 0600)
+	err := writeFile(filepath, []byte(contents), 0600)
 	require.NoError(t, err)
 	r.waitForWriters()
 }

docs/content/authors.md

@@ -316,3 +316,8 @@ Contributors
   * anuar45 <serdaliyev.anuar@gmail.com>
   * Fernando <ferferga@users.noreply.github.com>
   * David Cole <david.cole@sohonet.com>
+  * Wei He <git@weispot.com>
+  * Outvi V <19144373+outloudvi@users.noreply.github.com>
+  * Thomas Kriechbaumer <thomas@kriechbaumer.name>
+  * Tennix <tennix@users.noreply.github.com>
+  * Ole Schütt <ole@schuett.name>

docs/content/docs.md

@@ -37,7 +37,7 @@ See the following for detailed instructions for
   * [Google Photos](/googlephotos/)
   * [HTTP](/http/)
   * [Hubic](/hubic/)
-  * [Jottacloud](/jottacloud/)
+  * [Jottacloud / GetSky.no](/jottacloud/)
   * [Koofr](/koofr/)
   * [Mail.ru Cloud](/mailru/)
   * [Mega](/mega/)
@@ -770,6 +770,23 @@ in effect (the defaults):
 - 500MB..750MB files will be downloaded with 3 streams
 - 750MB+ files will be downloaded with 4 streams
 
+### --no-check-dest ###
+
+The `--no-check-dest` can be used with `move` or `copy` and it causes
+rclone not to check the destination at all when copying files.
+
+This means that:
+
+- the destination is not listed minimising the API calls
+- files are always transferred
+- this can cause duplicates on remotes which allow it (eg Google Drive)
+- `--retries 1` is recommended otherwise you'll transfer everything again on a retry
+
+This flag is useful to minimise the transactions if you know that none
+of the files are on the destination.
+
+This is a specialized flag which should be ignored by most users!
+
 ### --no-gzip-encoding ###
 
 Don't set `Accept-Encoding: gzip`.  This means that rclone won't ask

docs/content/s3.md

@@ -136,8 +136,11 @@ Choose a number from below, or type in your own value
    / Asia Pacific (Mumbai)
 13 | Needs location constraint ap-south-1.
    \ "ap-south-1"
+   / Asia Patific (Hong Kong) Region
+14 | Needs location constraint ap-east-1.
+   \ "ap-east-1"
    / South America (Sao Paulo) Region
-14 | Needs location constraint sa-east-1.
+15 | Needs location constraint sa-east-1.
    \ "sa-east-1"
 region> 1
 Endpoint for S3 API.
@@ -171,7 +174,9 @@ Choose a number from below, or type in your own value
    \ "ap-northeast-2"
 13 / Asia Pacific (Mumbai)
    \ "ap-south-1"
-14 / South America (Sao Paulo) Region.
+14 / Asia Pacific (Hong Kong)
+   \ "ap-east-1"
+15 / South America (Sao Paulo) Region.
    \ "sa-east-1"
 location_constraint> 1
 Canned ACL used when creating buckets and/or storing objects in S3.
@@ -265,8 +270,8 @@ The modified time is stored as metadata on the object as
 `X-Amz-Meta-Mtime` as floating point since the epoch accurate to 1 ns.
 
 If the modification time needs to be updated rclone will attempt to perform a server
-side copy to update the modification if the object can be copied in a single part.  
-In the case the object is larger than 5Gb or is in Glacier or Glacier Deep Archive 
+side copy to update the modification if the object can be copied in a single part.
+In the case the object is larger than 5Gb or is in Glacier or Glacier Deep Archive
 storage the object will be uploaded rather than copied.
 
 #### Restricted filename characters
@@ -353,6 +358,7 @@ The different authentication methods are tried in this order:
          - `AWS_PROFILE` to control which profile to use.
    - Or, run `rclone` in an ECS task with an IAM role (AWS only).
    - Or, run `rclone` on an EC2 instance with an IAM role (AWS only).
+   - Or, run `rclone` in an EKS pod with an IAM role that is associated with a service account (AWS only).
 
 If none of these option actually end up providing `rclone` with AWS
 credentials then S3 interaction will be non-authenticated (see below).
@@ -556,6 +562,9 @@ Region to connect to.
     - "ap-south-1"
         - Asia Pacific (Mumbai)
         - Needs location constraint ap-south-1.
+    - "ap-east-1"
+        - Asia Pacific (Hong Kong) Region
+        - Needs location constraint ap-east-1.
     - "sa-east-1"
         - South America (Sao Paulo) Region
         - Needs location constraint sa-east-1.
@@ -775,6 +784,8 @@ Used when creating buckets only.
         - Asia Pacific (Seoul)
     - "ap-south-1"
         - Asia Pacific (Mumbai)
+    - "ap-east-1"
+        - Asia Pacific (Hong Kong)
     - "sa-east-1"
         - South America (Sao Paulo) Region.
 
@@ -1124,6 +1135,21 @@ WARNING: Storing parts of an incomplete multipart upload counts towards space us
 - Type:        bool
 - Default:     false
 
+#### --s3-list-chunk
+
+Size of listing chunk (response list for each ListObject S3 request).
+
+This option is also known as "MaxKeys", "max-items", or "page-size" from the AWS S3 specification.
+Most services truncate the response list to 1000 objects even if requested more than that.
+In AWS S3 this is a global maximum and cannot be changed, see [AWS S3](https://docs.aws.amazon.com/cli/latest/reference/s3/ls.html).
+In Ceph, this can be increased with the "rgw list buckets max chunk" option.
+
+
+- Config:      list_chunk
+- Env Var:     RCLONE_S3_LIST_CHUNK
+- Type:        int
+- Default:     1000
+
 <!--- autogenerated options stop -->
 
 ### Anonymous access to public buckets ###

docs/static/css/custom.css

@@ -43,7 +43,11 @@ h3:hover .header-link,
 h4:hover .header-link,
 h5:hover .header-link,
 h6:hover .header-link {
-  opacity: 1;
+  	opacity: 1;
+}
+h2, h3, h4, h5, h6 {
+	padding-top: 55px;
+	margin-top: -44px;
 }
 
 /* Fix spacing between menu items */

fs/accounting/stats.go

@@ -634,20 +634,6 @@ func (s *StatsInfo) RemoveTransfer(transfer *Transfer) {
 	s.mu.Unlock()
 }
 
-// PruneAllTransfers removes all finished transfers.
-func (s *StatsInfo) PruneAllTransfers() {
-	s.mu.Lock()
-	for i := 0; i < len(s.startedTransfers); i++ {
-		tr := s.startedTransfers[i]
-		if tr.IsDone() {
-			s.removeTransfer(tr, i)
-			// i'th element is removed, recover iterator to not skip next element.
-			i--
-		}
-	}
-	s.mu.Unlock()
-}
-
 // PruneTransfers makes sure there aren't too many old transfers by removing
 // single finished transfer.
 func (s *StatsInfo) PruneTransfers() {

fs/accounting/stats_groups.go

@@ -13,7 +13,15 @@ const globalStats = "global_stats"
 
 var groups *statsGroups
 
-func listStats(ctx context.Context, in rc.Params) (rc.Params, error) {
+func init() {
+	// Init stats container
+	groups = newStatsGroups()
+
+	// Set the function pointer up in fs
+	fs.CountError = GlobalStats().Error
+}
+
+func rcListStats(ctx context.Context, in rc.Params) (rc.Params, error) {
 	out := make(rc.Params)
 
 	out["groups"] = groups.names()
@@ -21,7 +29,30 @@ func listStats(ctx context.Context, in rc.Params) (rc.Params, error) {
 	return out, nil
 }
 
-func remoteStats(ctx context.Context, in rc.Params) (rc.Params, error) {
+func init() {
+	rc.Add(rc.Call{
+		Path:  "core/group-list",
+		Fn:    rcListStats,
+		Title: "Returns list of stats.",
+		Help: `
+This returns list of stats groups currently in memory. 
+
+Returns the following values:
+` + "```" + `
+{
+	"groups":  an array of group names:
+		[
+			"group1",
+			"group2",
+			...
+		]
+}
+` + "```" + `
+`,
+	})
+}
+
+func rcRemoteStats(ctx context.Context, in rc.Params) (rc.Params, error) {
 	// Check to see if we should filter by group.
 	group, err := in.GetString("group")
 	if rc.NotErrParamNotFound(err) {
@@ -34,52 +65,10 @@ func remoteStats(ctx context.Context, in rc.Params) (rc.Params, error) {
 	return groups.sum().RemoteStats()
 }
 
-func transferredStats(ctx context.Context, in rc.Params) (rc.Params, error) {
-	// Check to see if we should filter by group.
-	group, err := in.GetString("group")
-	if rc.NotErrParamNotFound(err) {
-		return rc.Params{}, err
-	}
-
-	out := make(rc.Params)
-	if group != "" {
-		out["transferred"] = StatsGroup(group).Transferred()
-	} else {
-		out["transferred"] = groups.sum().Transferred()
-	}
-
-	return out, nil
-}
-
-func resetStats(ctx context.Context, in rc.Params) (rc.Params, error) {
-	// Check to see if we should filter by group.
-	group, err := in.GetString("group")
-	if rc.NotErrParamNotFound(err) {
-		return rc.Params{}, err
-	}
-
-	if group != "" {
-		stats := groups.get(group)
-		stats.ResetCounters()
-		stats.ResetErrors()
-		stats.PruneAllTransfers()
-	} else {
-		groups.clear()
-	}
-
-	return rc.Params{}, nil
-}
-
 func init() {
-	// Init stats container
-	groups = newStatsGroups()
-
-	// Set the function pointer up in fs
-	fs.CountError = GlobalStats().Error
-
 	rc.Add(rc.Call{
 		Path:  "core/stats",
-		Fn:    remoteStats,
+		Fn:    rcRemoteStats,
 		Title: "Returns stats about current transfers.",
 		Help: `
 This returns all available stats:
@@ -127,10 +116,29 @@ Values for "transferring", "checking" and "lastError" are only assigned if data
 The value for "eta" is null if an eta cannot be determined.
 `,
 	})
+}
 
+func rcTransferredStats(ctx context.Context, in rc.Params) (rc.Params, error) {
+	// Check to see if we should filter by group.
+	group, err := in.GetString("group")
+	if rc.NotErrParamNotFound(err) {
+		return rc.Params{}, err
+	}
+
+	out := make(rc.Params)
+	if group != "" {
+		out["transferred"] = StatsGroup(group).Transferred()
+	} else {
+		out["transferred"] = groups.sum().Transferred()
+	}
+
+	return out, nil
+}
+
+func init() {
 	rc.Add(rc.Call{
 		Path:  "core/transferred",
-		Fn:    transferredStats,
+		Fn:    rcTransferredStats,
 		Title: "Returns stats about completed transfers.",
 		Help: `
 This returns stats about completed transfers:
@@ -165,31 +173,30 @@ Returns the following values:
 ` + "```" + `
 `,
 	})
-
-	rc.Add(rc.Call{
-		Path:  "core/group-list",
-		Fn:    listStats,
-		Title: "Returns list of stats.",
-		Help: `
-This returns list of stats groups currently in memory. 
-
-Returns the following values:
-` + "```" + `
-{
-	"groups":  an array of group names:
-		[
-			"group1",
-			"group2",
-			...
-		]
 }
-` + "```" + `
-`,
-	})
 
+func rcResetStats(ctx context.Context, in rc.Params) (rc.Params, error) {
+	// Check to see if we should filter by group.
+	group, err := in.GetString("group")
+	if rc.NotErrParamNotFound(err) {
+		return rc.Params{}, err
+	}
+
+	if group != "" {
+		stats := groups.get(group)
+		stats.ResetErrors()
+		stats.ResetCounters()
+	} else {
+		groups.reset()
+	}
+
+	return rc.Params{}, nil
+}
+
+func init() {
 	rc.Add(rc.Call{
 		Path:  "core/stats-reset",
-		Fn:    resetStats,
+		Fn:    rcResetStats,
 		Title: "Reset stats.",
 		Help: `
 This clears counters, errors and finished transfers for all stats or specific 
@@ -202,6 +209,35 @@ Parameters
 	})
 }
 
+func rcDeleteStats(ctx context.Context, in rc.Params) (rc.Params, error) {
+	// Group name required because we only do single group.
+	group, err := in.GetString("group")
+	if rc.NotErrParamNotFound(err) {
+		return rc.Params{}, err
+	}
+
+	if group != "" {
+		groups.delete(group)
+	}
+
+	return rc.Params{}, nil
+}
+
+func init() {
+	rc.Add(rc.Call{
+		Path:  "core/stats-delete",
+		Fn:    rcDeleteStats,
+		Title: "Delete stats group.",
+		Help: `
+This deletes entire stats group
+
+Parameters
+
+- group - name of the stats group (string)
+`,
+	})
+}
+
 type statsGroupCtx int64
 
 const statsGroupKey statsGroupCtx = 1
@@ -274,13 +310,13 @@ func (sg *statsGroups) set(group string, stats *StatsInfo) {
 	// Limit number of groups kept in memory.
 	if len(sg.order) >= fs.Config.MaxStatsGroups {
 		group := sg.order[0]
-		fs.LogPrintf(fs.LogLevelInfo, nil, "Max number of stats groups reached removing %s", group)
+		//fs.LogPrintf(fs.LogLevelInfo, nil, "Max number of stats groups reached removing %s", group)
 		delete(sg.m, group)
 		r := (len(sg.order) - fs.Config.MaxStatsGroups) + 1
 		sg.order = sg.order[r:]
 	}
 
-	// Exclude global stats from
+	// Exclude global stats from listing
 	if group != globalStats {
 		sg.order = append(sg.order, group)
 	}
@@ -304,40 +340,66 @@ func (sg *statsGroups) names() []string {
 	return sg.order
 }
 
-// get gets the stats for group, or nil if not found
+// sum returns aggregate stats that contains summation of all groups.
 func (sg *statsGroups) sum() *StatsInfo {
 	sg.mu.Lock()
 	defer sg.mu.Unlock()
+
 	sum := NewStats()
 	for _, stats := range sg.m {
-		sum.bytes += stats.bytes
-		sum.errors += stats.errors
-		sum.fatalError = sum.fatalError || stats.fatalError
-		sum.retryError = sum.retryError || stats.retryError
-		sum.checks += stats.checks
-		sum.transfers += stats.transfers
-		sum.deletes += stats.deletes
-		sum.checking.merge(stats.checking)
-		sum.transferring.merge(stats.transferring)
-		sum.inProgress.merge(stats.inProgress)
-		if sum.lastError == nil && stats.lastError != nil {
-			sum.lastError = stats.lastError
+		stats.mu.RLock()
+		{
+			sum.bytes += stats.bytes
+			sum.errors += stats.errors
+			sum.fatalError = sum.fatalError || stats.fatalError
+			sum.retryError = sum.retryError || stats.retryError
+			sum.checks += stats.checks
+			sum.transfers += stats.transfers
+			sum.deletes += stats.deletes
+			sum.checking.merge(stats.checking)
+			sum.transferring.merge(stats.transferring)
+			sum.inProgress.merge(stats.inProgress)
+			if sum.lastError == nil && stats.lastError != nil {
+				sum.lastError = stats.lastError
+			}
+			sum.startedTransfers = append(sum.startedTransfers, stats.startedTransfers...)
 		}
-		sum.startedTransfers = append(sum.startedTransfers, stats.startedTransfers...)
+		stats.mu.RUnlock()
 	}
 	return sum
 }
 
-func (sg *statsGroups) clear() {
+func (sg *statsGroups) reset() {
 	sg.mu.Lock()
 	defer sg.mu.Unlock()
 
 	for _, stats := range sg.m {
 		stats.ResetErrors()
 		stats.ResetCounters()
-		stats.PruneAllTransfers()
 	}
 
 	sg.m = make(map[string]*StatsInfo)
 	sg.order = nil
 }
+
+// delete removes all references to the group.
+func (sg *statsGroups) delete(group string) {
+	sg.mu.Lock()
+	defer sg.mu.Unlock()
+	stats := sg.m[group]
+	if stats == nil {
+		return
+	}
+	stats.ResetErrors()
+	stats.ResetCounters()
+	delete(sg.m, group)
+
+	// Remove group reference from the ordering slice.
+	tmp := sg.order[:0]
+	for _, g := range sg.order {
+		if g != group {
+			tmp = append(tmp, g)
+		}
+	}
+	sg.order = tmp
+}

fs/accounting/stats_groups_test.go

@@ -0,0 +1,104 @@
+package accounting
+
+import (
+	"fmt"
+	"runtime"
+	"testing"
+)
+
+func TestStatsGroupOperations(t *testing.T) {
+
+	t.Run("empty group returns nil", func(t *testing.T) {
+		t.Parallel()
+		sg := newStatsGroups()
+		sg.get("invalid-group")
+	})
+
+	t.Run("set assigns stats to group", func(t *testing.T) {
+		t.Parallel()
+		stats := NewStats()
+		sg := newStatsGroups()
+		sg.set("test", stats)
+		sg.set("test1", stats)
+		if len(sg.m) != len(sg.names()) || len(sg.m) != 2 {
+			t.Fatalf("Expected two stats got %d, %d", len(sg.m), len(sg.order))
+		}
+	})
+
+	t.Run("get returns correct group", func(t *testing.T) {
+		t.Parallel()
+		stats := NewStats()
+		sg := newStatsGroups()
+		sg.set("test", stats)
+		sg.set("test1", stats)
+		got := sg.get("test")
+		if got != stats {
+			t.Fatal("get returns incorrect stats")
+		}
+	})
+
+	t.Run("sum returns correct values", func(t *testing.T) {
+		t.Parallel()
+		stats1 := NewStats()
+		stats1.bytes = 5
+		stats1.errors = 5
+		stats2 := NewStats()
+		sg := newStatsGroups()
+		sg.set("test1", stats1)
+		sg.set("test2", stats2)
+		sum := sg.sum()
+		if sum.bytes != stats1.bytes+stats2.bytes {
+			t.Fatalf("sum() => bytes %d, expected %d", sum.bytes, stats1.bytes+stats2.bytes)
+		}
+		if sum.errors != stats1.errors+stats2.errors {
+			t.Fatalf("sum() => errors %d, expected %d", sum.errors, stats1.errors+stats2.errors)
+		}
+	})
+
+	t.Run("delete removes stats", func(t *testing.T) {
+		t.Parallel()
+		stats := NewStats()
+		sg := newStatsGroups()
+		sg.set("test", stats)
+		sg.set("test1", stats)
+		sg.delete("test1")
+		if sg.get("test1") != nil {
+			t.Fatal("stats not deleted")
+		}
+		if len(sg.m) != len(sg.names()) || len(sg.m) != 1 {
+			t.Fatalf("Expected two stats got %d, %d", len(sg.m), len(sg.order))
+		}
+	})
+
+	t.Run("memory is reclaimed", func(t *testing.T) {
+		var (
+			count      = 1000
+			start, end runtime.MemStats
+			sg         = newStatsGroups()
+		)
+
+		runtime.GC()
+		runtime.ReadMemStats(&start)
+
+		for i := 0; i < count; i++ {
+			sg.set(fmt.Sprintf("test-%d", i), NewStats())
+		}
+
+		for i := 0; i < count; i++ {
+			sg.delete(fmt.Sprintf("test-%d", i))
+		}
+
+		runtime.GC()
+		runtime.ReadMemStats(&end)
+
+		t.Log(fmt.Sprintf("%+v\n%+v", start, end))
+		diff := percentDiff(start.HeapObjects, end.HeapObjects)
+		if diff > 1 || diff < 0 {
+			t.Errorf("HeapObjects = %d, expected %d", end.HeapObjects, start.HeapObjects)
+		}
+	})
+}
+
+func percentDiff(start, end uint64) uint64 {
+	return (start - end) * 100 / start
+}

fs/accounting/stats_test.go

@@ -431,25 +431,3 @@ func TestPruneTransfers(t *testing.T) {
 		})
 	}
 }
-
-func TestPruneAllTransfers(t *testing.T) {
-	const transfers = 10
-
-	s := NewStats()
-	for i := int64(1); i <= int64(transfers); i++ {
-		s.AddTransfer(&Transfer{
-			startedAt:   time.Unix(i, 0),
-			completedAt: time.Unix(i+1, 0),
-		})
-	}
-
-	s.mu.Lock()
-	assert.Equal(t, transfers, len(s.startedTransfers))
-	s.mu.Unlock()
-
-	s.PruneAllTransfers()
-
-	s.mu.Lock()
-	assert.Empty(t, s.startedTransfers)
-	s.mu.Unlock()
-}

fs/config.go

@@ -67,6 +67,7 @@ type ConfigInfo struct {
 	IgnoreChecksum         bool
 	IgnoreCaseSync         bool
 	NoTraverse             bool
+	NoCheckDest            bool
 	NoUpdateModTime        bool
 	DataRateUnit           string
 	CompareDest            string

fs/config/configflags/configflags.go

@@ -68,6 +68,7 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &fs.Config.IgnoreChecksum, "ignore-checksum", "", fs.Config.IgnoreChecksum, "Skip post copy check of checksums.")
 	flags.BoolVarP(flagSet, &fs.Config.IgnoreCaseSync, "ignore-case-sync", "", fs.Config.IgnoreCaseSync, "Ignore case when synchronizing")
 	flags.BoolVarP(flagSet, &fs.Config.NoTraverse, "no-traverse", "", fs.Config.NoTraverse, "Don't traverse destination file system on copy.")
+	flags.BoolVarP(flagSet, &fs.Config.NoCheckDest, "no-check-dest", "", fs.Config.NoCheckDest, "Don't check the destination, copy regardless.")
 	flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.")
 	flags.StringVarP(flagSet, &fs.Config.CompareDest, "compare-dest", "", fs.Config.CompareDest, "Include additional server-side path during comparison.")
 	flags.StringVarP(flagSet, &fs.Config.CopyDest, "copy-dest", "", fs.Config.CopyDest, "Implies --compare-dest but also copies files from path into destination.")

fs/log/log.go

@@ -10,16 +10,24 @@ import (
 	"strings"
 
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config/flags"
 )
 
-// Flags
-var (
-	logFile        = flags.StringP("log-file", "", "", "Log everything to this file")
-	logFormat      = flags.StringP("log-format", "", "date,time", "Comma separated list of log format options")
-	useSyslog      = flags.BoolP("syslog", "", false, "Use Syslog for logging")
-	syslogFacility = flags.StringP("syslog-facility", "", "DAEMON", "Facility for syslog, eg KERN,USER,...")
-)
+// Options contains options for the remote control server
+type Options struct {
+	File           string // Log everything to this file
+	Format         string // Comma separated list of log format options
+	UseSyslog      bool   // Use Syslog for logging
+	SyslogFacility string // Facility for syslog, eg KERN,USER,...
+}
+
+// DefaultOpt is the default values used for Opt
+var DefaultOpt = Options{
+	Format:         "date,time",
+	SyslogFacility: "DAEMON",
+}
+
+// Opt is the options for the logger
+var Opt = DefaultOpt
 
 // fnName returns the name of the calling +2 function
 func fnName() string {
@@ -79,7 +87,7 @@ func Stack(o interface{}, info string) {
 
 // InitLogging start the logging as per the command line flags
 func InitLogging() {
-	flagsStr := "," + *logFormat + ","
+	flagsStr := "," + Opt.Format + ","
 	var flags int
 	if strings.Contains(flagsStr, ",date,") {
 		flags |= log.Ldate
@@ -102,8 +110,8 @@ func InitLogging() {
 	log.SetFlags(flags)
 
 	// Log file output
-	if *logFile != "" {
-		f, err := os.OpenFile(*logFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)
+	if Opt.File != "" {
+		f, err := os.OpenFile(Opt.File, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640)
 		if err != nil {
 			log.Fatalf("Failed to open log file: %v", err)
 		}
@@ -116,8 +124,8 @@ func InitLogging() {
 	}
 
 	// Syslog output
-	if *useSyslog {
-		if *logFile != "" {
+	if Opt.UseSyslog {
+		if Opt.File != "" {
 			log.Fatalf("Can't use --syslog and --log-file together")
 		}
 		startSysLog()
@@ -126,5 +134,5 @@ func InitLogging() {
 
 // Redirected returns true if the log has been redirected from stdout
 func Redirected() bool {
-	return *useSyslog || *logFile != ""
+	return Opt.UseSyslog || Opt.File != ""
 }

fs/log/logflags/logflags.go

@@ -0,0 +1,19 @@
+// Package logflags implements command line flags to set up the log
+package logflags
+
+import (
+	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/rclone/rclone/fs/log"
+	"github.com/rclone/rclone/fs/rc"
+	"github.com/spf13/pflag"
+)
+
+// AddFlags adds the log flags to the flagSet
+func AddFlags(flagSet *pflag.FlagSet) {
+	rc.AddOption("log", &log.Opt)
+
+	flags.StringVarP(flagSet, &log.Opt.File, "log-file", "", log.Opt.File, "Log everything to this file")
+	flags.StringVarP(flagSet, &log.Opt.Format, "log-format", "", log.Opt.Format, "Comma separated list of log format options")
+	flags.BoolVarP(flagSet, &log.Opt.UseSyslog, "syslog", "", log.Opt.UseSyslog, "Use Syslog for logging")
+	flags.StringVarP(flagSet, &log.Opt.SyslogFacility, "syslog-facility", "", log.Opt.SyslogFacility, "Facility for syslog, eg KERN,USER,...")
+}

fs/log/syslog_unix.go

@@ -32,9 +32,9 @@ var (
 
 // Starts syslog
 func startSysLog() bool {
-	facility, ok := syslogFacilityMap[*syslogFacility]
+	facility, ok := syslogFacilityMap[Opt.SyslogFacility]
 	if !ok {
-		log.Fatalf("Unknown syslog facility %q - man syslog for list", *syslogFacility)
+		log.Fatalf("Unknown syslog facility %q - man syslog for list", Opt.SyslogFacility)
 	}
 	Me := path.Base(os.Args[0])
 	w, err := syslog.New(syslog.LOG_NOTICE|facility, Me)

fs/march/march.go

@@ -30,6 +30,7 @@ type March struct {
 	SrcIncludeAll bool            // don't include all files in the src
 	DstIncludeAll bool            // don't include all files in the destination
 	Callback      Marcher         // object to call with results
+	NoCheckDest   bool            // transfer all objects regardless without checking dst
 	// internal state
 	srcListDir listDirFn // function to call to list a directory in the src
 	dstListDir listDirFn // function to call to list a directory in the dst
@@ -188,6 +189,7 @@ func (m *March) Run() error {
 		srcDepth:  srcDepth - 1,
 		dstRemote: m.Dir,
 		dstDepth:  dstDepth - 1,
+		noDst:     m.NoCheckDest,
 	}
 	go func() {
 		// when the context is cancelled discard the remaining jobs
@@ -406,7 +408,7 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
 
 	// If NoTraverse is set, then try to find a matching object
 	// for each item in the srcList
-	if m.NoTraverse {
+	if m.NoTraverse && !m.NoCheckDest {
 		for _, src := range srcList {
 			if srcObj, ok := src.(fs.Object); ok {
 				leaf := path.Base(srcObj.Remote())

fs/operations/dedupe.go

@@ -15,7 +15,6 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/walk"
-	"github.com/spf13/pflag"
 )
 
 // dedupeRename renames the objs slice to different names
@@ -196,9 +195,6 @@ func (x *DeduplicateMode) Type() string {
 	return "string"
 }
 
-// Check it satisfies the interface
-var _ pflag.Value = (*DeduplicateMode)(nil)
-
 // dedupeFindDuplicateDirs scans f for duplicate directories
 func dedupeFindDuplicateDirs(ctx context.Context, f fs.Fs) ([][]fs.Directory, error) {
 	dirs := map[string][]fs.Directory{}

fs/operations/dedupe_test.go

@@ -10,10 +10,14 @@ import (
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/fstest"
+	"github.com/spf13/pflag"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+// Check flag satisfies the interface
+var _ pflag.Value = (*operations.DeduplicateMode)(nil)
+
 func skipIfCantDedupe(t *testing.T, f fs.Fs) {
 	if !f.Features().DuplicateFiles {
 		t.Skip("Can't test deduplicate - no duplicate files possible")

fs/operations/operations.go

@@ -402,6 +402,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
 		// Retry if err returned a retry error
 		if fserrors.IsRetryError(err) || fserrors.ShouldRetry(err) {
 			fs.Debugf(src, "Received error: %v - low level retry %d/%d", err, tries, maxTries)
+			tr.Reset() // skip incomplete accounting - will be overwritten by retry
 			continue
 		}
 		// otherwise finish
@@ -778,10 +779,12 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b
 				atomic.AddInt32(&c.differences, 1)
 			} else {
 				atomic.AddInt32(&c.matches, 1)
-				fs.Debugf(dstX, "OK")
-			}
-			if noHash {
-				atomic.AddInt32(&c.noHashes, 1)
+				if noHash {
+					atomic.AddInt32(&c.noHashes, 1)
+					fs.Debugf(dstX, "OK - could not check hash")
+				} else {
+					fs.Debugf(dstX, "OK")
+				}
 			}
 		} else {
 			err := errors.Errorf("is file on %v but directory on %v", c.fsrc, c.fdst)
@@ -1705,11 +1708,14 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
 	}
 
 	// Find dst object if it exists
-	dstObj, err := fdst.NewObject(ctx, dstFileName)
-	if err == fs.ErrorObjectNotFound {
-		dstObj = nil
-	} else if err != nil {
-		return err
+	var dstObj fs.Object
+	if !fs.Config.NoCheckDest {
+		dstObj, err = fdst.NewObject(ctx, dstFileName)
+		if err == fs.ErrorObjectNotFound {
+			dstObj = nil
+		} else if err != nil {
+			return err
+		}
 	}
 
 	// Special case for changing case of a file on a case insensitive remote

fs/sync/sync.go

@@ -31,6 +31,7 @@ type syncCopyMove struct {
 	ctx             context.Context        // internal context for controlling go-routines
 	cancel          func()                 // cancel the context
 	noTraverse      bool                   // if set don't traverse the dst
+	noCheckDest     bool                   // if set transfer all objects regardless without checking dst
 	deletersWg      sync.WaitGroup         // for delete before go routine
 	deleteFilesCh   chan fs.Object         // channel to receive deletes if delete before
 	trackRenames    bool                   // set if we should do server side renames
@@ -82,6 +83,7 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 		dstEmptyDirs:       make(map[string]fs.DirEntry),
 		srcEmptyDirs:       make(map[string]fs.DirEntry),
 		noTraverse:         fs.Config.NoTraverse,
+		noCheckDest:        fs.Config.NoCheckDest,
 		toBeChecked:        newPipe(accounting.Stats(ctx).SetCheckQueue, fs.Config.MaxBacklog),
 		toBeUploaded:       newPipe(accounting.Stats(ctx).SetTransferQueue, fs.Config.MaxBacklog),
 		deleteFilesCh:      make(chan fs.Object, fs.Config.Checkers),
@@ -95,6 +97,17 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
 		fs.Errorf(nil, "Ignoring --no-traverse with sync")
 		s.noTraverse = false
 	}
+	if s.noCheckDest {
+		if s.deleteMode != fs.DeleteModeOff {
+			return nil, errors.New("can't use --no-check-dest with sync: use copy instead")
+		}
+		if fs.Config.Immutable {
+			return nil, errors.New("can't use --no-check-dest with --immutable")
+		}
+		if s.backupDir != nil {
+			return nil, errors.New("can't use --no-check-dest with --backup-dir")
+		}
+	}
 	if s.trackRenames {
 		// Don't track renames for remotes without server-side move support.
 		if !operations.CanServerSideMove(fdst) {
@@ -667,6 +680,7 @@ func (s *syncCopyMove) run() error {
 		NoTraverse:    s.noTraverse,
 		Callback:      s,
 		DstIncludeAll: filter.Active.Opt.DeleteExcluded,
+		NoCheckDest:   s.noCheckDest,
 	}
 	s.processError(m.Run())
 

fstest/fstests/fstests.go

@@ -18,6 +18,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"sort"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -1571,36 +1572,39 @@ func Run(t *testing.T, opt *Opt) {
 					t.Skip("FS has no PutStream interface")
 				}
 
-				file := fstest.Item{
-					ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
-					Path:    "piped data.txt",
-					Size:    -1, // use unknown size during upload
+				for _, contentSize := range []int{0, 100} {
+					t.Run(strconv.Itoa(contentSize), func(t *testing.T) {
+						file := fstest.Item{
+							ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
+							Path:    "piped data.txt",
+							Size:    -1, // use unknown size during upload
+						}
+
+						var (
+							err        error
+							obj        fs.Object
+							uploadHash *hash.MultiHasher
+						)
+						retry(t, "PutStream", func() error {
+							contents := random.String(contentSize)
+							buf := bytes.NewBufferString(contents)
+							uploadHash = hash.NewMultiHasher()
+							in := io.TeeReader(buf, uploadHash)
+
+							file.Size = -1
+							obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
+							obj, err = remote.Features().PutStream(ctx, in, obji)
+							return err
+						})
+						file.Hashes = uploadHash.Sums()
+						file.Size = int64(contentSize) // use correct size when checking
+						file.Check(t, obj, remote.Precision())
+						// Re-read the object and check again
+						obj = findObject(ctx, t, remote, file.Path)
+						file.Check(t, obj, remote.Precision())
+						require.NoError(t, obj.Remove(ctx))
+					})
 				}
-
-				var (
-					err         error
-					obj         fs.Object
-					uploadHash  *hash.MultiHasher
-					contentSize = 100
-				)
-				retry(t, "PutStream", func() error {
-					contents := random.String(contentSize)
-					buf := bytes.NewBufferString(contents)
-					uploadHash = hash.NewMultiHasher()
-					in := io.TeeReader(buf, uploadHash)
-
-					file.Size = -1
-					obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
-					obj, err = remote.Features().PutStream(ctx, in, obji)
-					return err
-				})
-				file.Hashes = uploadHash.Sums()
-				file.Size = int64(contentSize) // use correct size when checking
-				file.Check(t, obj, remote.Precision())
-				// Re-read the object and check again
-				obj = findObject(ctx, t, remote, file.Path)
-				file.Check(t, obj, remote.Precision())
-				require.NoError(t, obj.Remove(ctx))
 			})
 
 			// TestInternal calls InternalTest() on the Fs

go.mod

@@ -1,7 +1,7 @@
 module github.com/rclone/rclone
 
 require (
-	bazil.org/fuse v0.0.0-20180421153158-65cc252bf669
+	bazil.org/fuse v0.0.0-20191225233854-3a99aca11732
 	cloud.google.com/go v0.47.0 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.2
 	github.com/Azure/azure-storage-blob-go v0.8.0
@@ -61,7 +61,7 @@ require (
 	golang.org/x/net v0.0.0-20191109021931-daa7c04131f5
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
-	golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056
+	golang.org/x/sys v0.0.0-20191210023423-ac6580df4449
 	golang.org/x/text v0.3.2
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	google.golang.org/api v0.13.0

go.sum

@@ -1,5 +1,7 @@
 bazil.org/fuse v0.0.0-20180421153158-65cc252bf669 h1:FNCRpXiquG1aoyqcIWVFmpTSKVcx2bQD38uZZeGtdlw=
 bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
+bazil.org/fuse v0.0.0-20191225233854-3a99aca11732 h1:gaB1+kZCJDExjlrdy37gIwxV0M7v81EzIFKQZ5o5YV0=
+bazil.org/fuse v0.0.0-20191225233854-3a99aca11732/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@@ -254,6 +256,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/t3rm1n4l/go-mega v0.0.0-20191014094753-e8695d78299a h1:9VwG6wBA1jd6oOCnmQ/OaKM1GRfChadtH5N3bx1oSKE=
 github.com/t3rm1n4l/go-mega v0.0.0-20191014094753-e8695d78299a/go.mod h1:XWL4vDyd3JKmJx+hZWUVgCNmmhZ2dTBcaNDcxH465s0=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
@@ -346,6 +349,8 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056 h1:dHtDnRWQtSx0Hjq9kvKFpBh9uPPKfQN70NZZmvssGwk=
 golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ=
+golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=

vendor/bazil.org/fuse/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2013-2015 Tommi Virtanen.
+Copyright (c) 2013-2019 Tommi Virtanen.
 Copyright (c) 2009, 2011, 2012 The Go Authors.
 All rights reserved.
 

vendor/bazil.org/fuse/fs/serve.go

@@ -3,6 +3,8 @@
 package fs // import "bazil.org/fuse/fs"
 
 import (
+	"bytes"
+	"context"
 	"encoding/binary"
 	"fmt"
 	"hash/fnv"
@@ -14,12 +16,6 @@ import (
 	"sync"
 	"time"
 
-	"golang.org/x/net/context"
-)
-
-import (
-	"bytes"
-
 	"bazil.org/fuse"
 	"bazil.org/fuse/fuseutil"
 )
@@ -71,6 +67,9 @@ type FSInodeGenerator interface {
 	//
 	// Implementing this is useful to e.g. constrain the range of
 	// inode values used for dynamic inodes.
+	//
+	// Non-zero return values should be greater than 1, as that is
+	// always used for the root inode.
 	GenerateInode(parentInode uint64, name string) uint64
 }
 
@@ -435,8 +434,6 @@ func Serve(c *fuse.Conn, fs FS) error {
 	return server.Serve(fs)
 }
 
-type nothing struct{}
-
 type serveRequest struct {
 	Request fuse.Request
 	cancel  func()
@@ -1557,11 +1554,12 @@ func GenerateDynamicInode(parent uint64, name string) uint64 {
 	var inode uint64
 	for {
 		inode = h.Sum64()
-		if inode != 0 {
+		if inode > 1 {
 			break
 		}
-		// there's a tiny probability that result is zero; change the
-		// input a little and try again
+		// there's a tiny probability that result is zero or the
+		// hardcoded root inode 1; change the input a little and try
+		// again
 		_, _ = h.Write([]byte{'x'})
 	}
 	return inode

vendor/bazil.org/fuse/fs/tree.go

@@ -3,14 +3,11 @@
 package fs
 
 import (
+	"context"
 	"os"
 	pathpkg "path"
 	"strings"
 
-	"golang.org/x/net/context"
-)
-
-import (
 	"bazil.org/fuse"
 )
 

vendor/bazil.org/fuse/fuse.go

@@ -495,9 +495,15 @@ func fileMode(unixMode uint32) os.FileMode {
 		mode |= os.ModeSymlink
 	case syscall.S_IFSOCK:
 		mode |= os.ModeSocket
+	case 0:
+		// apparently there's plenty of times when the FUSE request
+		// does not contain the file type
+		mode |= os.ModeIrregular
 	default:
-		// no idea
-		mode |= os.ModeDevice
+		// not just unavailable in the kernel codepath; known to
+		// kernel but unrecognized by us
+		Debug(fmt.Sprintf("unrecognized file mode type: %04o", unixMode))
+		mode |= os.ModeIrregular
 	}
 	if unixMode&syscall.S_ISUID != 0 {
 		mode |= os.ModeSetuid
@@ -1004,7 +1010,12 @@ loop:
 		}
 
 	case opBmap:
-		panic("opBmap")
+		// bmap asks to map a byte offset within a file to a single
+		// uint64. On Linux, it triggers only with blkdev fuse mounts,
+		// that claim to be backed by an actual block device. FreeBSD
+		// seems to send it for just any fuse mount, whether there's a
+		// block device involved or not.
+		goto unrecognized
 
 	case opDestroy:
 		req = &DestroyRequest{
@@ -1384,8 +1395,6 @@ func (a *Attr) attr(out *attr, proto Protocol) {
 	if proto.GE(Protocol{7, 9}) {
 		out.Blksize = a.BlockSize
 	}
-
-	return
 }
 
 // A GetattrRequest asks for the metadata for the file denoted by r.Node.
@@ -1464,7 +1473,7 @@ type GetxattrResponse struct {
 }
 
 func (r *GetxattrResponse) String() string {
-	return fmt.Sprintf("Getxattr %x", r.Xattr)
+	return fmt.Sprintf("Getxattr %q", r.Xattr)
 }
 
 // A ListxattrRequest asks to list the extended attributes associated with r.Node.
@@ -1500,7 +1509,7 @@ type ListxattrResponse struct {
 }
 
 func (r *ListxattrResponse) String() string {
-	return fmt.Sprintf("Listxattr %x", r.Xattr)
+	return fmt.Sprintf("Listxattr %q", r.Xattr)
 }
 
 // Append adds an extended attribute name to the response.
@@ -1565,7 +1574,7 @@ func trunc(b []byte, max int) ([]byte, string) {
 
 func (r *SetxattrRequest) String() string {
 	xattr, tail := trunc(r.Xattr, 16)
-	return fmt.Sprintf("Setxattr [%s] %q %x%s fl=%v @%#x", &r.Header, r.Name, xattr, tail, r.Flags, r.Position)
+	return fmt.Sprintf("Setxattr [%s] %q %q%s fl=%v @%#x", &r.Header, r.Name, xattr, tail, r.Flags, r.Position)
 }
 
 // Respond replies to the request, indicating that the extended attribute was set.
@@ -1986,9 +1995,14 @@ type SetattrRequest struct {
 	Size   uint64
 	Atime  time.Time
 	Mtime  time.Time
-	Mode   os.FileMode
-	Uid    uint32
-	Gid    uint32
+	// Mode is the file mode to set (when valid).
+	//
+	// The type of the node (as in os.ModeType, os.ModeDir etc) is not
+	// guaranteed to be sent by the kernel, in which case
+	// os.ModeIrregular will be set.
+	Mode os.FileMode
+	Uid  uint32
+	Gid  uint32
 
 	// OS X only
 	Bkuptime time.Time

vendor/bazil.org/fuse/fuse_kernel.go

@@ -232,7 +232,7 @@ type OpenResponseFlags uint32
 const (
 	OpenDirectIO    OpenResponseFlags = 1 << 0 // bypass page cache for this open file
 	OpenKeepCache   OpenResponseFlags = 1 << 1 // don't invalidate the data cache on open
-	OpenNonSeekable OpenResponseFlags = 1 << 2 // mark the file as non-seekable (not supported on OS X)
+	OpenNonSeekable OpenResponseFlags = 1 << 2 // mark the file as non-seekable (not supported on OS X or FreeBSD)
 
 	OpenPurgeAttr OpenResponseFlags = 1 << 30 // OS X
 	OpenPurgeUBC  OpenResponseFlags = 1 << 31 // OS X

vendor/bazil.org/fuse/fuse_kernel_linux.go

@@ -18,7 +18,7 @@ type attr struct {
 	Gid       uint32
 	Rdev      uint32
 	Blksize   uint32
-	padding   uint32
+	_         uint32
 }
 
 func (a *attr) Crtime() time.Time {

vendor/bazil.org/fuse/go.mod

@@ -0,0 +1,8 @@
+module bazil.org/fuse
+
+go 1.13
+
+require (
+	github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c
+	golang.org/x/sys v0.0.0-20191210023423-ac6580df4449
+)

vendor/bazil.org/fuse/go.sum

@@ -0,0 +1,4 @@
+github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
+github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
+golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ=
+golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

vendor/bazil.org/fuse/options.go

@@ -106,6 +106,14 @@ func NoAppleXattr() MountOption {
 	return noAppleXattr
 }
 
+// NoBrowse makes OSXFUSE mark the volume as non-browsable, so that
+// Finder won't automatically browse it.
+//
+// OS X only.  Others ignore this option.
+func NoBrowse() MountOption {
+	return noBrowse
+}
+
 // ExclCreate causes O_EXCL flag to be set for only "truly" exclusive creates,
 // i.e. create calls for which the initiator explicitly set the O_EXCL flag.
 //
@@ -155,7 +163,8 @@ func AllowOther() MountOption {
 	}
 }
 
-// AllowRoot allows other users to access the file system.
+// AllowRoot allows root (but not just everyone) to access the file
+// system.
 //
 // Only one of AllowOther or AllowRoot can be used.
 //

vendor/bazil.org/fuse/options_darwin.go

@@ -33,3 +33,8 @@ func exclCreate(conf *mountConfig) error {
 	conf.options["excl_create"] = ""
 	return nil
 }
+
+func noBrowse(conf *mountConfig) error {
+	conf.options["nobrowse"] = ""
+	return nil
+}

vendor/bazil.org/fuse/options_freebsd.go

@@ -26,3 +26,7 @@ func noAppleDouble(conf *mountConfig) error {
 func exclCreate(conf *mountConfig) error {
 	return nil
 }
+
+func noBrowse(conf *mountConfig) error {
+	return nil
+}

vendor/bazil.org/fuse/options_linux.go

@@ -23,3 +23,7 @@ func noAppleDouble(conf *mountConfig) error {
 func exclCreate(conf *mountConfig) error {
 	return nil
 }
+
+func noBrowse(conf *mountConfig) error {
+	return nil
+}

vendor/github.com/t3rm1n4l/go-mega/mega.go

@@ -704,11 +704,20 @@ func (m *Mega) addFSNode(itm FSNode) (*Node, error) {
 	switch {
 	case itm.T == FOLDER || itm.T == FILE:
 		args := strings.Split(itm.Key, ":")
+		if len(args) < 2 {
+			return nil, fmt.Errorf("not enough : in item.Key: %q", itm.Key)
+		}
+		itemUser, itemKey := args[0], args[1]
+		itemKeyParts := strings.Split(itemKey, "/")
+		if len(itemKeyParts) >= 2 {
+			itemKey = itemKeyParts[0]
+			// the other part is maybe a share key handle?
+		}
 
 		switch {
 		// File or folder owned by current user
-		case args[0] == itm.User:
-			buf, err := base64urldecode(args[1])
+		case itemUser == itm.User:
+			buf, err := base64urldecode(itemKey)
 			if err != nil {
 				return nil, err
 			}
@@ -736,7 +745,7 @@ func (m *Mega) addFSNode(itm FSNode) (*Node, error) {
 			}
 
 			m.FS.skmap[itm.Hash] = itm.SKey
-			buf, err := base64urldecode(args[1])
+			buf, err := base64urldecode(itemKey)
 			if err != nil {
 				return nil, err
 			}
@@ -750,7 +759,7 @@ func (m *Mega) addFSNode(itm FSNode) (*Node, error) {
 			}
 			// Shared file
 		default:
-			k := m.FS.skmap[args[0]]
+			k := m.FS.skmap[itemUser]
 			b, err := base64urldecode(k)
 			if err != nil {
 				return nil, err
@@ -763,7 +772,7 @@ func (m *Mega) addFSNode(itm FSNode) (*Node, error) {
 			if err != nil {
 				return nil, err
 			}
-			buf, err := base64urldecode(args[1])
+			buf, err := base64urldecode(itemKey)
 			if err != nil {
 				return nil, err
 			}

vendor/golang.org/x/sys/cpu/cpu_arm64.go

@@ -0,0 +1,134 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cpu
+
+import "runtime"
+
+const cacheLineSize = 64
+
+func init() {
+	switch runtime.GOOS {
+	case "darwin":
+		// iOS does not seem to allow reading these registers
+	case "android", "linux":
+		doinit()
+	default:
+		readARM64Registers()
+	}
+}
+
+func readARM64Registers() {
+	Initialized = true
+
+	// ID_AA64ISAR0_EL1
+	isar0 := getisar0()
+
+	switch extractBits(isar0, 4, 7) {
+	case 1:
+		ARM64.HasAES = true
+	case 2:
+		ARM64.HasAES = true
+		ARM64.HasPMULL = true
+	}
+
+	switch extractBits(isar0, 8, 11) {
+	case 1:
+		ARM64.HasSHA1 = true
+	}
+
+	switch extractBits(isar0, 12, 15) {
+	case 1:
+		ARM64.HasSHA2 = true
+	case 2:
+		ARM64.HasSHA2 = true
+		ARM64.HasSHA512 = true
+	}
+
+	switch extractBits(isar0, 16, 19) {
+	case 1:
+		ARM64.HasCRC32 = true
+	}
+
+	switch extractBits(isar0, 20, 23) {
+	case 2:
+		ARM64.HasATOMICS = true
+	}
+
+	switch extractBits(isar0, 28, 31) {
+	case 1:
+		ARM64.HasASIMDRDM = true
+	}
+
+	switch extractBits(isar0, 32, 35) {
+	case 1:
+		ARM64.HasSHA3 = true
+	}
+
+	switch extractBits(isar0, 36, 39) {
+	case 1:
+		ARM64.HasSM3 = true
+	}
+
+	switch extractBits(isar0, 40, 43) {
+	case 1:
+		ARM64.HasSM4 = true
+	}
+
+	switch extractBits(isar0, 44, 47) {
+	case 1:
+		ARM64.HasASIMDDP = true
+	}
+
+	// ID_AA64ISAR1_EL1
+	isar1 := getisar1()
+
+	switch extractBits(isar1, 0, 3) {
+	case 1:
+		ARM64.HasDCPOP = true
+	}
+
+	switch extractBits(isar1, 12, 15) {
+	case 1:
+		ARM64.HasJSCVT = true
+	}
+
+	switch extractBits(isar1, 16, 19) {
+	case 1:
+		ARM64.HasFCMA = true
+	}
+
+	switch extractBits(isar1, 20, 23) {
+	case 1:
+		ARM64.HasLRCPC = true
+	}
+
+	// ID_AA64PFR0_EL1
+	pfr0 := getpfr0()
+
+	switch extractBits(pfr0, 16, 19) {
+	case 0:
+		ARM64.HasFP = true
+	case 1:
+		ARM64.HasFP = true
+		ARM64.HasFPHP = true
+	}
+
+	switch extractBits(pfr0, 20, 23) {
+	case 0:
+		ARM64.HasASIMD = true
+	case 1:
+		ARM64.HasASIMD = true
+		ARM64.HasASIMDHP = true
+	}
+
+	switch extractBits(pfr0, 32, 35) {
+	case 1:
+		ARM64.HasSVE = true
+	}
+}
+
+func extractBits(data uint64, start, end uint) uint {
+	return (uint)(data>>start) & ((1 << (end - start + 1)) - 1)
+}

vendor/golang.org/x/sys/cpu/cpu_arm64.s

@@ -0,0 +1,31 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+#include "textflag.h"
+
+// func getisar0() uint64
+TEXT ·getisar0(SB),NOSPLIT,$0-8
+	// get Instruction Set Attributes 0 into x0
+	// mrs x0, ID_AA64ISAR0_EL1 = d5380600
+	WORD	$0xd5380600
+	MOVD	R0, ret+0(FP)
+	RET
+
+// func getisar1() uint64
+TEXT ·getisar1(SB),NOSPLIT,$0-8
+	// get Instruction Set Attributes 1 into x0
+	// mrs x0, ID_AA64ISAR1_EL1 = d5380620
+	WORD	$0xd5380620
+	MOVD	R0, ret+0(FP)
+	RET
+
+// func getpfr0() uint64
+TEXT ·getpfr0(SB),NOSPLIT,$0-8
+	// get Processor Feature Register 0 into x0
+	// mrs x0, ID_AA64PFR0_EL1 = d5380400
+	WORD	$0xd5380400
+	MOVD	R0, ret+0(FP)
+	RET

vendor/golang.org/x/sys/cpu/cpu_gc_arm64.go

@@ -0,0 +1,11 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !gccgo
+
+package cpu
+
+func getisar0() uint64
+func getisar1() uint64
+func getpfr0() uint64

vendor/golang.org/x/sys/cpu/cpu_gccgo_arm64.go

@@ -0,0 +1,11 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build gccgo
+
+package cpu
+
+func getisar0() uint64 { return 0 }
+func getisar1() uint64 { return 0 }
+func getpfr0() uint64  { return 0 }

vendor/golang.org/x/sys/cpu/cpu_linux.go

@@ -2,58 +2,14 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !amd64,!amd64p32,!386
+// +build !386,!amd64,!amd64p32,!arm64
 
 package cpu
 
-import (
-	"io/ioutil"
-)
-
-const (
-	_AT_HWCAP  = 16
-	_AT_HWCAP2 = 26
-
-	procAuxv = "/proc/self/auxv"
-
-	uintSize = int(32 << (^uint(0) >> 63))
-)
-
-// For those platforms don't have a 'cpuid' equivalent we use HWCAP/HWCAP2
-// These are initialized in cpu_$GOARCH.go
-// and should not be changed after they are initialized.
-var hwCap uint
-var hwCap2 uint
-
 func init() {
-	buf, err := ioutil.ReadFile(procAuxv)
-	if err != nil {
-		// e.g. on android /proc/self/auxv is not accessible, so silently
-		// ignore the error and leave Initialized = false
+	if err := readHWCAP(); err != nil {
 		return
 	}
-
-	bo := hostByteOrder()
-	for len(buf) >= 2*(uintSize/8) {
-		var tag, val uint
-		switch uintSize {
-		case 32:
-			tag = uint(bo.Uint32(buf[0:]))
-			val = uint(bo.Uint32(buf[4:]))
-			buf = buf[8:]
-		case 64:
-			tag = uint(bo.Uint64(buf[0:]))
-			val = uint(bo.Uint64(buf[8:]))
-			buf = buf[16:]
-		}
-		switch tag {
-		case _AT_HWCAP:
-			hwCap = val
-		case _AT_HWCAP2:
-			hwCap2 = val
-		}
-	}
 	doinit()
-
 	Initialized = true
 }

vendor/golang.org/x/sys/cpu/cpu_linux_arm64.go

@@ -4,8 +4,6 @@
 
 package cpu
 
-const cacheLineSize = 64
-
 // HWCAP/HWCAP2 bits. These are exposed by Linux.
 const (
 	hwcap_FP       = 1 << 0
@@ -35,6 +33,12 @@ const (
 )
 
 func doinit() {
+	if err := readHWCAP(); err != nil {
+		// failed to read /proc/self/auxv, try reading registers directly
+		readARM64Registers()
+		return
+	}
+
 	// HWCAP feature bits
 	ARM64.HasFP = isSet(hwCap, hwcap_FP)
 	ARM64.HasASIMD = isSet(hwCap, hwcap_ASIMD)

vendor/golang.org/x/sys/cpu/cpu_other_arm64.go

@@ -6,4 +6,4 @@
 
 package cpu
 
-const cacheLineSize = 64
+func doinit() {}

vendor/golang.org/x/sys/cpu/hwcap_linux.go

@@ -0,0 +1,56 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cpu
+
+import (
+	"io/ioutil"
+)
+
+const (
+	_AT_HWCAP  = 16
+	_AT_HWCAP2 = 26
+
+	procAuxv = "/proc/self/auxv"
+
+	uintSize = int(32 << (^uint(0) >> 63))
+)
+
+// For those platforms don't have a 'cpuid' equivalent we use HWCAP/HWCAP2
+// These are initialized in cpu_$GOARCH.go
+// and should not be changed after they are initialized.
+var hwCap uint
+var hwCap2 uint
+
+func readHWCAP() error {
+	buf, err := ioutil.ReadFile(procAuxv)
+	if err != nil {
+		// e.g. on android /proc/self/auxv is not accessible, so silently
+		// ignore the error and leave Initialized = false. On some
+		// architectures (e.g. arm64) doinit() implements a fallback
+		// readout and will set Initialized = true again.
+		return err
+	}
+	bo := hostByteOrder()
+	for len(buf) >= 2*(uintSize/8) {
+		var tag, val uint
+		switch uintSize {
+		case 32:
+			tag = uint(bo.Uint32(buf[0:]))
+			val = uint(bo.Uint32(buf[4:]))
+			buf = buf[8:]
+		case 64:
+			tag = uint(bo.Uint64(buf[0:]))
+			val = uint(bo.Uint64(buf[8:]))
+			buf = buf[16:]
+		}
+		switch tag {
+		case _AT_HWCAP:
+			hwCap = val
+		case _AT_HWCAP2:
+			hwCap2 = val
+		}
+	}
+	return nil
+}

vendor/golang.org/x/sys/unix/mkall.sh

@@ -50,7 +50,7 @@ if [[ "$GOOS" = "linux" ]]; then
 	# Use the Docker-based build system
 	# Files generated through docker (use $cmd so you can Ctl-C the build or run)
 	$cmd docker build --tag generate:$GOOS $GOOS
-	$cmd docker run --interactive --tty --volume $(dirname "$(readlink -f "$0")"):/build generate:$GOOS
+	$cmd docker run --interactive --tty --volume $(cd -- "$(dirname -- "$0")" && /bin/pwd):/build generate:$GOOS
 	exit
 fi
 

vendor/golang.org/x/sys/unix/mkerrors.sh

@@ -44,6 +44,7 @@ includes_AIX='
 #include <sys/stropts.h>
 #include <sys/mman.h>
 #include <sys/poll.h>
+#include <sys/select.h>
 #include <sys/termio.h>
 #include <termios.h>
 #include <fcntl.h>
@@ -196,6 +197,7 @@ struct ltchars {
 #include <linux/fanotify.h>
 #include <linux/filter.h>
 #include <linux/fs.h>
+#include <linux/fscrypt.h>
 #include <linux/genetlink.h>
 #include <linux/hdreg.h>
 #include <linux/icmpv6.h>
@@ -495,7 +497,9 @@ ccflags="$@"
 		$2 ~ /^CAN_/ ||
 		$2 ~ /^CAP_/ ||
 		$2 ~ /^ALG_/ ||
-		$2 ~ /^FS_(POLICY_FLAGS|KEY_DESC|ENCRYPTION_MODE|[A-Z0-9_]+_KEY_SIZE|IOC_(GET|SET)_ENCRYPTION)/ ||
+		$2 ~ /^FS_(POLICY_FLAGS|KEY_DESC|ENCRYPTION_MODE|[A-Z0-9_]+_KEY_SIZE)/ ||
+		$2 ~ /^FS_IOC_.*ENCRYPTION/ ||
+		$2 ~ /^FSCRYPT_/ ||
 		$2 ~ /^GRND_/ ||
 		$2 ~ /^RND/ ||
 		$2 ~ /^KEY_(SPEC|REQKEY_DEFL)_/ ||

vendor/golang.org/x/sys/unix/syscall_netbsd.go

@@ -249,6 +249,14 @@ func Sendfile(outfd int, infd int, offset *int64, count int) (written int, err e
 	return sendfile(outfd, infd, offset, count)
 }
 
+func Fstatvfs(fd int, buf *Statvfs_t) (err error) {
+	return Fstatvfs1(fd, buf, ST_WAIT)
+}
+
+func Statvfs(path string, buf *Statvfs_t) (err error) {
+	return Statvfs1(path, buf, ST_WAIT)
+}
+
 /*
  * Exposed directly
  */
@@ -287,6 +295,7 @@ func Sendfile(outfd int, infd int, offset *int64, count int) (written int, err e
 //sys	Fpathconf(fd int, name int) (val int, err error)
 //sys	Fstat(fd int, stat *Stat_t) (err error)
 //sys	Fstatat(fd int, path string, stat *Stat_t, flags int) (err error)
+//sys	Fstatvfs1(fd int, buf *Statvfs_t) (err error) = SYS_FSTATVFS1
 //sys	Fsync(fd int) (err error)
 //sys	Ftruncate(fd int, length int64) (err error)
 //sysnb	Getegid() (egid int)
@@ -343,6 +352,7 @@ func Sendfile(outfd int, infd int, offset *int64, count int) (written int, err e
 //sysnb	Settimeofday(tp *Timeval) (err error)
 //sysnb	Setuid(uid int) (err error)
 //sys	Stat(path string, stat *Stat_t) (err error)
+//sys	Statvfs1(path string, buf *Statvfs_t) (err error) = SYS_STATVFS1
 //sys	Symlink(path string, link string) (err error)
 //sys	Symlinkat(oldpath string, newdirfd int, newpath string) (err error)
 //sys	Sync() (err error)

vendor/golang.org/x/sys/unix/zerrors_aix_ppc.go

@@ -459,6 +459,15 @@ const (
 	MAP_SHARED                    = 0x1
 	MAP_TYPE                      = 0xf0
 	MAP_VARIABLE                  = 0x0
+	MCAST_BLOCK_SOURCE            = 0x40
+	MCAST_EXCLUDE                 = 0x2
+	MCAST_INCLUDE                 = 0x1
+	MCAST_JOIN_GROUP              = 0x3e
+	MCAST_JOIN_SOURCE_GROUP       = 0x42
+	MCAST_LEAVE_GROUP             = 0x3f
+	MCAST_LEAVE_SOURCE_GROUP      = 0x43
+	MCAST_SOURCE_FILTER           = 0x49
+	MCAST_UNBLOCK_SOURCE          = 0x41
 	MCL_CURRENT                   = 0x100
 	MCL_FUTURE                    = 0x200
 	MSG_ANY                       = 0x4
@@ -483,6 +492,7 @@ const (
 	MS_INVALIDATE                 = 0x40
 	MS_PER_SEC                    = 0x3e8
 	MS_SYNC                       = 0x20
+	NFDBITS                       = 0x20
 	NL0                           = 0x0
 	NL1                           = 0x4000
 	NL2                           = 0x8000
@@ -688,7 +698,7 @@ const (
 	SIOCGHIWAT                    = 0x40047301
 	SIOCGIFADDR                   = -0x3fd796df
 	SIOCGIFADDRS                  = 0x2000698c
-	SIOCGIFBAUDRATE               = -0x3fd79693
+	SIOCGIFBAUDRATE               = -0x3fdf9669
 	SIOCGIFBRDADDR                = -0x3fd796dd
 	SIOCGIFCONF                   = -0x3ff796bb
 	SIOCGIFCONFGLOB               = -0x3ff79670

vendor/golang.org/x/sys/unix/zerrors_aix_ppc64.go

@@ -459,6 +459,15 @@ const (
 	MAP_SHARED                    = 0x1
 	MAP_TYPE                      = 0xf0
 	MAP_VARIABLE                  = 0x0
+	MCAST_BLOCK_SOURCE            = 0x40
+	MCAST_EXCLUDE                 = 0x2
+	MCAST_INCLUDE                 = 0x1
+	MCAST_JOIN_GROUP              = 0x3e
+	MCAST_JOIN_SOURCE_GROUP       = 0x42
+	MCAST_LEAVE_GROUP             = 0x3f
+	MCAST_LEAVE_SOURCE_GROUP      = 0x43
+	MCAST_SOURCE_FILTER           = 0x49
+	MCAST_UNBLOCK_SOURCE          = 0x41
 	MCL_CURRENT                   = 0x100
 	MCL_FUTURE                    = 0x200
 	MSG_ANY                       = 0x4
@@ -483,6 +492,7 @@ const (
 	MS_INVALIDATE                 = 0x40
 	MS_PER_SEC                    = 0x3e8
 	MS_SYNC                       = 0x20
+	NFDBITS                       = 0x40
 	NL0                           = 0x0
 	NL1                           = 0x4000
 	NL2                           = 0x8000
@@ -688,7 +698,7 @@ const (
 	SIOCGHIWAT                    = 0x40047301
 	SIOCGIFADDR                   = -0x3fd796df
 	SIOCGIFADDRS                  = 0x2000698c
-	SIOCGIFBAUDRATE               = -0x3fd79693
+	SIOCGIFBAUDRATE               = -0x3fdf9669
 	SIOCGIFBRDADDR                = -0x3fd796dd
 	SIOCGIFCONF                   = -0x3fef96bb
 	SIOCGIFCONFGLOB               = -0x3fef9670

vendor/golang.org/x/sys/unix/zptrace_armnn_linux.go

@@ -1,4 +1,4 @@
-// Code generated by linux/mkall.go generatePtracePair(arm, arm64). DO NOT EDIT.
+// Code generated by linux/mkall.go generatePtracePair("arm", "arm64"). DO NOT EDIT.
 
 // +build linux
 // +build arm arm64

vendor/golang.org/x/sys/unix/zptrace_linux_arm64.go

@@ -0,0 +1,17 @@
+// Code generated by linux/mkall.go generatePtraceRegSet("arm64"). DO NOT EDIT.
+
+package unix
+
+import "unsafe"
+
+// PtraceGetRegSetArm64 fetches the registers used by arm64 binaries.
+func PtraceGetRegSetArm64(pid, addr int, regsout *PtraceRegsArm64) error {
+	iovec := Iovec{(*byte)(unsafe.Pointer(regsout)), uint64(unsafe.Sizeof(*regsout))}
+	return ptrace(PTRACE_GETREGSET, pid, uintptr(addr), uintptr(unsafe.Pointer(&iovec)))
+}
+
+// PtraceSetRegSetArm64 sets the registers used by arm64 binaries.
+func PtraceSetRegSetArm64(pid, addr int, regs *PtraceRegsArm64) error {
+	iovec := Iovec{(*byte)(unsafe.Pointer(regs)), uint64(unsafe.Sizeof(*regs))}
+	return ptrace(PTRACE_SETREGSET, pid, uintptr(addr), uintptr(unsafe.Pointer(&iovec)))
+}

vendor/golang.org/x/sys/unix/zptrace_mipsnn_linux.go

@@ -1,4 +1,4 @@
-// Code generated by linux/mkall.go generatePtracePair(mips, mips64). DO NOT EDIT.
+// Code generated by linux/mkall.go generatePtracePair("mips", "mips64"). DO NOT EDIT.
 
 // +build linux
 // +build mips mips64

vendor/golang.org/x/sys/unix/zptrace_mipsnnle_linux.go

@@ -1,4 +1,4 @@
-// Code generated by linux/mkall.go generatePtracePair(mipsle, mips64le). DO NOT EDIT.
+// Code generated by linux/mkall.go generatePtracePair("mipsle", "mips64le"). DO NOT EDIT.
 
 // +build linux
 // +build mipsle mips64le

vendor/golang.org/x/sys/unix/zptrace_x86_linux.go

@@ -1,4 +1,4 @@
-// Code generated by linux/mkall.go generatePtracePair(386, amd64). DO NOT EDIT.
+// Code generated by linux/mkall.go generatePtracePair("386", "amd64"). DO NOT EDIT.
 
 // +build linux
 // +build 386 amd64

vendor/golang.org/x/sys/unix/zsyscall_netbsd_386.go

@@ -926,6 +926,16 @@ func Fstatat(fd int, path string, stat *Stat_t, flags int) (err error) {
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Fstatvfs1(fd int, buf *Statvfs_t, flags int) (err error) {
+	_, _, e1 := Syscall(SYS_FSTATVFS1, uintptr(fd), uintptr(unsafe.Pointer(buf)), uintptr(flags))
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Fsync(fd int) (err error) {
 	_, _, e1 := Syscall(SYS_FSYNC, uintptr(fd), 0, 0)
 	if e1 != 0 {
@@ -1635,6 +1645,21 @@ func Stat(path string, stat *Stat_t) (err error) {
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Statvfs1(path string, buf *Statvfs_t, flags int) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	_, _, e1 := Syscall(SYS_STATVFS1, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(buf)), uintptr(flags))
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Symlink(path string, link string) (err error) {
 	var _p0 *byte
 	_p0, err = BytePtrFromString(path)

vendor/golang.org/x/sys/unix/zsyscall_netbsd_amd64.go

@@ -926,6 +926,16 @@ func Fstatat(fd int, path string, stat *Stat_t, flags int) (err error) {
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Fstatvfs1(fd int, buf *Statvfs_t, flags int) (err error) {
+	_, _, e1 := Syscall(SYS_FSTATVFS1, uintptr(fd), uintptr(unsafe.Pointer(buf)), uintptr(flags))
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Fsync(fd int) (err error) {
 	_, _, e1 := Syscall(SYS_FSYNC, uintptr(fd), 0, 0)
 	if e1 != 0 {
@@ -1635,6 +1645,21 @@ func Stat(path string, stat *Stat_t) (err error) {
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Statvfs1(path string, buf *Statvfs_t, flags int) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	_, _, e1 := Syscall(SYS_STATVFS1, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(buf)), uintptr(flags))
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Symlink(path string, link string) (err error) {
 	var _p0 *byte
 	_p0, err = BytePtrFromString(path)

vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm.go

@@ -926,6 +926,16 @@ func Fstatat(fd int, path string, stat *Stat_t, flags int) (err error) {
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Fstatvfs1(fd int, buf *Statvfs_t, flags int) (err error) {
+	_, _, e1 := Syscall(SYS_FSTATVFS1, uintptr(fd), uintptr(unsafe.Pointer(buf)), uintptr(flags))
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Fsync(fd int) (err error) {
 	_, _, e1 := Syscall(SYS_FSYNC, uintptr(fd), 0, 0)
 	if e1 != 0 {
@@ -1635,6 +1645,21 @@ func Stat(path string, stat *Stat_t) (err error) {
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Statvfs1(path string, buf *Statvfs_t, flags int) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	_, _, e1 := Syscall(SYS_STATVFS1, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(buf)), uintptr(flags))
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Symlink(path string, link string) (err error) {
 	var _p0 *byte
 	_p0, err = BytePtrFromString(path)

vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm64.go

@@ -926,6 +926,16 @@ func Fstatat(fd int, path string, stat *Stat_t, flags int) (err error) {
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Fstatvfs1(fd int, buf *Statvfs_t, flags int) (err error) {
+	_, _, e1 := Syscall(SYS_FSTATVFS1, uintptr(fd), uintptr(unsafe.Pointer(buf)), uintptr(flags))
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Fsync(fd int) (err error) {
 	_, _, e1 := Syscall(SYS_FSYNC, uintptr(fd), 0, 0)
 	if e1 != 0 {
@@ -1635,6 +1645,21 @@ func Stat(path string, stat *Stat_t) (err error) {
 
 // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
 
+func Statvfs1(path string, buf *Statvfs_t, flags int) (err error) {
+	var _p0 *byte
+	_p0, err = BytePtrFromString(path)
+	if err != nil {
+		return
+	}
+	_, _, e1 := Syscall(SYS_STATVFS1, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(buf)), uintptr(flags))
+	if e1 != 0 {
+		err = errnoErr(e1)
+	}
+	return
+}
+
+// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
+
 func Symlink(path string, link string) (err error) {
 	var _p0 *byte
 	_p0, err = BytePtrFromString(path)

vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go

@@ -415,4 +415,5 @@ const (
 	SYS_FSMOUNT                      = 4432
 	SYS_FSPICK                       = 4433
 	SYS_PIDFD_OPEN                   = 4434
+	SYS_CLONE3                       = 4435
 )

vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go

@@ -345,4 +345,5 @@ const (
 	SYS_FSMOUNT                = 5432
 	SYS_FSPICK                 = 5433
 	SYS_PIDFD_OPEN             = 5434
+	SYS_CLONE3                 = 5435
 )

vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go

@@ -345,4 +345,5 @@ const (
 	SYS_FSMOUNT                = 5432
 	SYS_FSPICK                 = 5433
 	SYS_PIDFD_OPEN             = 5434
+	SYS_CLONE3                 = 5435
 )

vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go

@@ -415,4 +415,5 @@ const (
 	SYS_FSMOUNT                      = 4432
 	SYS_FSPICK                       = 4433
 	SYS_PIDFD_OPEN                   = 4434
+	SYS_CLONE3                       = 4435
 )

vendor/golang.org/x/sys/unix/ztypes_linux_386.go

@@ -179,6 +179,55 @@ type FscryptKey struct {
 	Size uint32
 }
 
+type FscryptPolicyV1 struct {
+	Version                   uint8
+	Contents_encryption_mode  uint8
+	Filenames_encryption_mode uint8
+	Flags                     uint8
+	Master_key_descriptor     [8]uint8
+}
+
+type FscryptPolicyV2 struct {
+	Version                   uint8
+	Contents_encryption_mode  uint8
+	Filenames_encryption_mode uint8
+	Flags                     uint8
+	_                         [4]uint8
+	Master_key_identifier     [16]uint8
+}
+
+type FscryptGetPolicyExArg struct {
+	Size   uint64
+	Policy [24]byte
+}
+
+type FscryptKeySpecifier struct {
+	Type uint32
+	_    uint32
+	U    [32]byte
+}
+
+type FscryptAddKeyArg struct {
+	Key_spec FscryptKeySpecifier
+	Raw_size uint32
+	_        [9]uint32
+}
+
+type FscryptRemoveKeyArg struct {
+	Key_spec             FscryptKeySpecifier
+	Removal_status_flags uint32
+	_                    [5]uint32
+}
+
+type FscryptGetKeyStatusArg struct {
+	Key_spec     FscryptKeySpecifier
+	_            [6]uint32
+	Status       uint32
+	Status_flags uint32
+	User_count   uint32
+	_            [13]uint32
+}
+
 type KeyctlDHParams struct {
 	Private int32
 	Prime   int32
@@ -256,7 +305,7 @@ type RawSockaddrRFCOMM struct {
 type RawSockaddrCAN struct {
 	Family  uint16
 	Ifindex int32
-	Addr    [8]byte
+	Addr    [16]byte
 }
 
 type RawSockaddrALG struct {
@@ -427,7 +476,7 @@ const (
 	SizeofSockaddrHCI       = 0x6
 	SizeofSockaddrL2        = 0xe
 	SizeofSockaddrRFCOMM    = 0xa
-	SizeofSockaddrCAN       = 0x10
+	SizeofSockaddrCAN       = 0x18
 	SizeofSockaddrALG       = 0x58
 	SizeofSockaddrVM        = 0x10
 	SizeofSockaddrXDP       = 0x10
@@ -2041,6 +2090,7 @@ type XDPRingOffset struct {
 	Producer uint64
 	Consumer uint64
 	Desc     uint64
+	Flags    uint64
 }
 
 type XDPMmapOffsets struct {
@@ -2055,6 +2105,7 @@ type XDPUmemReg struct {
 	Len      uint64
 	Size     uint32
 	Headroom uint32
+	Flags    uint32
 }
 
 type XDPStatistics struct {
@@ -2663,7 +2714,7 @@ const (
 	DEVLINK_CMD_DPIPE_ENTRIES_GET             = 0x20
 	DEVLINK_CMD_DPIPE_HEADERS_GET             = 0x21
 	DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET      = 0x22
-	DEVLINK_CMD_MAX                           = 0x3c
+	DEVLINK_CMD_MAX                           = 0x44
 	DEVLINK_PORT_TYPE_NOTSET                  = 0x0
 	DEVLINK_PORT_TYPE_AUTO                    = 0x1
 	DEVLINK_PORT_TYPE_ETH                     = 0x2
@@ -2743,7 +2794,7 @@ const (
 	DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE     = 0x3c
 	DEVLINK_ATTR_PAD                          = 0x3d
 	DEVLINK_ATTR_ESWITCH_ENCAP_MODE           = 0x3e
-	DEVLINK_ATTR_MAX                          = 0x80
+	DEVLINK_ATTR_MAX                          = 0x89
 	DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE     = 0x0
 	DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX  = 0x1
 	DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT      = 0x0

vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go

@@ -179,6 +179,55 @@ type FscryptKey struct {
 	Size uint32
 }
 
+type FscryptPolicyV1 struct {
+	Version                   uint8
+	Contents_encryption_mode  uint8
+	Filenames_encryption_mode uint8
+	Flags                     uint8
+	Master_key_descriptor     [8]uint8
+}
+
+type FscryptPolicyV2 struct {
+	Version                   uint8
+	Contents_encryption_mode  uint8
+	Filenames_encryption_mode uint8
+	Flags                     uint8
+	_                         [4]uint8
+	Master_key_identifier     [16]uint8
+}
+
+type FscryptGetPolicyExArg struct {
+	Size   uint64
+	Policy [24]byte
+}
+
+type FscryptKeySpecifier struct {
+	Type uint32
+	_    uint32
+	U    [32]byte
+}
+
+type FscryptAddKeyArg struct {
+	Key_spec FscryptKeySpecifier
+	Raw_size uint32
+	_        [9]uint32
+}
+
+type FscryptRemoveKeyArg struct {
+	Key_spec             FscryptKeySpecifier
+	Removal_status_flags uint32
+	_                    [5]uint32
+}
+
+type FscryptGetKeyStatusArg struct {
+	Key_spec     FscryptKeySpecifier
+	_            [6]uint32
+	Status       uint32
+	Status_flags uint32
+	User_count   uint32
+	_            [13]uint32
+}
+
 type KeyctlDHParams struct {
 	Private int32
 	Prime   int32
@@ -256,7 +305,7 @@ type RawSockaddrRFCOMM struct {
 type RawSockaddrCAN struct {
 	Family  uint16
 	Ifindex int32
-	Addr    [8]byte
+	Addr    [16]byte
 }
 
 type RawSockaddrALG struct {
@@ -428,7 +477,7 @@ const (
 	SizeofSockaddrHCI       = 0x6
 	SizeofSockaddrL2        = 0xe
 	SizeofSockaddrRFCOMM    = 0xa
-	SizeofSockaddrCAN       = 0x10
+	SizeofSockaddrCAN       = 0x18
 	SizeofSockaddrALG       = 0x58
 	SizeofSockaddrVM        = 0x10
 	SizeofSockaddrXDP       = 0x10
@@ -2054,6 +2103,7 @@ type XDPRingOffset struct {
 	Producer uint64
 	Consumer uint64
 	Desc     uint64
+	Flags    uint64
 }
 
 type XDPMmapOffsets struct {
@@ -2068,6 +2118,8 @@ type XDPUmemReg struct {
 	Len      uint64
 	Size     uint32
 	Headroom uint32
+	Flags    uint32
+	_        [4]byte
 }
 
 type XDPStatistics struct {
@@ -2677,7 +2729,7 @@ const (
 	DEVLINK_CMD_DPIPE_ENTRIES_GET             = 0x20
 	DEVLINK_CMD_DPIPE_HEADERS_GET             = 0x21
 	DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET      = 0x22
-	DEVLINK_CMD_MAX                           = 0x3c
+	DEVLINK_CMD_MAX                           = 0x44
 	DEVLINK_PORT_TYPE_NOTSET                  = 0x0
 	DEVLINK_PORT_TYPE_AUTO                    = 0x1
 	DEVLINK_PORT_TYPE_ETH                     = 0x2
@@ -2757,7 +2809,7 @@ const (
 	DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE     = 0x3c
 	DEVLINK_ATTR_PAD                          = 0x3d
 	DEVLINK_ATTR_ESWITCH_ENCAP_MODE           = 0x3e
-	DEVLINK_ATTR_MAX                          = 0x80
+	DEVLINK_ATTR_MAX                          = 0x89
 	DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE     = 0x0
 	DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX  = 0x1
 	DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT      = 0x0

vendor/golang.org/x/sys/unix/ztypes_linux_arm.go

@@ -183,6 +183,55 @@ type FscryptKey struct {
 	Size uint32
 }
 
+type FscryptPolicyV1 struct {
+	Version                   uint8
+	Contents_encryption_mode  uint8
+	Filenames_encryption_mode uint8
+	Flags                     uint8
+	Master_key_descriptor     [8]uint8
+}
+
+type FscryptPolicyV2 struct {
+	Version                   uint8
+	Contents_encryption_mode  uint8
+	Filenames_encryption_mode uint8
+	Flags                     uint8
+	_                         [4]uint8
+	Master_key_identifier     [16]uint8
+}
+
+type FscryptGetPolicyExArg struct {
+	Size   uint64
+	Policy [24]byte
+}
+
+type FscryptKeySpecifier struct {
+	Type uint32
+	_    uint32
+	U    [32]byte
+}
+
+type FscryptAddKeyArg struct {
+	Key_spec FscryptKeySpecifier
+	Raw_size uint32
+	_        [9]uint32
+}
+
+type FscryptRemoveKeyArg struct {
+	Key_spec             FscryptKeySpecifier
+	Removal_status_flags uint32
+	_                    [5]uint32
+}
+
+type FscryptGetKeyStatusArg struct {
+	Key_spec     FscryptKeySpecifier
+	_            [6]uint32
+	Status       uint32
+	Status_flags uint32
+	User_count   uint32
+	_            [13]uint32
+}
+
 type KeyctlDHParams struct {
 	Private int32
 	Prime   int32
@@ -260,7 +309,7 @@ type RawSockaddrRFCOMM struct {
 type RawSockaddrCAN struct {
 	Family  uint16
 	Ifindex int32
-	Addr    [8]byte
+	Addr    [16]byte
 }
 
 type RawSockaddrALG struct {
@@ -431,7 +480,7 @@ const (
 	SizeofSockaddrHCI       = 0x6
 	SizeofSockaddrL2        = 0xe
 	SizeofSockaddrRFCOMM    = 0xa
-	SizeofSockaddrCAN       = 0x10
+	SizeofSockaddrCAN       = 0x18
 	SizeofSockaddrALG       = 0x58
 	SizeofSockaddrVM        = 0x10
 	SizeofSockaddrXDP       = 0x10
@@ -2032,6 +2081,7 @@ type XDPRingOffset struct {
 	Producer uint64
 	Consumer uint64
 	Desc     uint64
+	Flags    uint64
 }
 
 type XDPMmapOffsets struct {
@@ -2046,6 +2096,8 @@ type XDPUmemReg struct {
 	Len      uint64
 	Size     uint32
 	Headroom uint32
+	Flags    uint32
+	_        [4]byte
 }
 
 type XDPStatistics struct {
@@ -2654,7 +2706,7 @@ const (
 	DEVLINK_CMD_DPIPE_ENTRIES_GET             = 0x20
 	DEVLINK_CMD_DPIPE_HEADERS_GET             = 0x21
 	DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET      = 0x22
-	DEVLINK_CMD_MAX                           = 0x3c
+	DEVLINK_CMD_MAX                           = 0x44
 	DEVLINK_PORT_TYPE_NOTSET                  = 0x0
 	DEVLINK_PORT_TYPE_AUTO                    = 0x1
 	DEVLINK_PORT_TYPE_ETH                     = 0x2
@@ -2734,7 +2786,7 @@ const (
 	DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE     = 0x3c
 	DEVLINK_ATTR_PAD                          = 0x3d
 	DEVLINK_ATTR_ESWITCH_ENCAP_MODE           = 0x3e
-	DEVLINK_ATTR_MAX                          = 0x80
+	DEVLINK_ATTR_MAX                          = 0x89
 	DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE     = 0x0
 	DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX  = 0x1
 	DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT      = 0x0

vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go

@@ -180,6 +180,55 @@ type FscryptKey struct {
 	Size uint32
 }
 
+type FscryptPolicyV1 struct {
+	Version                   uint8
+	Contents_encryption_mode  uint8
+	Filenames_encryption_mode uint8
+	Flags                     uint8
+	Master_key_descriptor     [8]uint8
+}
+
+type FscryptPolicyV2 struct {
+	Version                   uint8
+	Contents_encryption_mode  uint8
+	Filenames_encryption_mode uint8
+	Flags                     uint8
+	_                         [4]uint8
+	Master_key_identifier     [16]uint8
+}
+
+type FscryptGetPolicyExArg struct {
+	Size   uint64
+	Policy [24]byte
+}
+
+type FscryptKeySpecifier struct {
+	Type uint32
+	_    uint32
+	U    [32]byte
+}
+
+type FscryptAddKeyArg struct {
+	Key_spec FscryptKeySpecifier
+	Raw_size uint32
+	_        [9]uint32
+}
+
+type FscryptRemoveKeyArg struct {
+	Key_spec             FscryptKeySpecifier
+	Removal_status_flags uint32
+	_                    [5]uint32
+}
+
+type FscryptGetKeyStatusArg struct {
+	Key_spec     FscryptKeySpecifier
+	_            [6]uint32
+	Status       uint32
+	Status_flags uint32
+	User_count   uint32
+	_            [13]uint32
+}
+
 type KeyctlDHParams struct {
 	Private int32
 	Prime   int32
@@ -257,7 +306,7 @@ type RawSockaddrRFCOMM struct {
 type RawSockaddrCAN struct {
 	Family  uint16
 	Ifindex int32
-	Addr    [8]byte
+	Addr    [16]byte
 }
 
 type RawSockaddrALG struct {
@@ -429,7 +478,7 @@ const (
 	SizeofSockaddrHCI       = 0x6
 	SizeofSockaddrL2        = 0xe
 	SizeofSockaddrRFCOMM    = 0xa
-	SizeofSockaddrCAN       = 0x10
+	SizeofSockaddrCAN       = 0x18
 	SizeofSockaddrALG       = 0x58
 	SizeofSockaddrVM        = 0x10
 	SizeofSockaddrXDP       = 0x10
@@ -2033,6 +2082,7 @@ type XDPRingOffset struct {
 	Producer uint64
 	Consumer uint64
 	Desc     uint64
+	Flags    uint64
 }
 
 type XDPMmapOffsets struct {
@@ -2047,6 +2097,8 @@ type XDPUmemReg struct {
 	Len      uint64
 	Size     uint32
 	Headroom uint32
+	Flags    uint32
+	_        [4]byte
 }
 
 type XDPStatistics struct {
@@ -2656,7 +2708,7 @@ const (
 	DEVLINK_CMD_DPIPE_ENTRIES_GET             = 0x20
 	DEVLINK_CMD_DPIPE_HEADERS_GET             = 0x21
 	DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET      = 0x22
-	DEVLINK_CMD_MAX                           = 0x3c
+	DEVLINK_CMD_MAX                           = 0x44
 	DEVLINK_PORT_TYPE_NOTSET                  = 0x0
 	DEVLINK_PORT_TYPE_AUTO                    = 0x1
 	DEVLINK_PORT_TYPE_ETH                     = 0x2
@@ -2736,7 +2788,7 @@ const (
 	DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE     = 0x3c
 	DEVLINK_ATTR_PAD                          = 0x3d
 	DEVLINK_ATTR_ESWITCH_ENCAP_MODE           = 0x3e
-	DEVLINK_ATTR_MAX                          = 0x80
+	DEVLINK_ATTR_MAX                          = 0x89
 	DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE     = 0x0
 	DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX  = 0x1
 	DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT      = 0x0

vendor/golang.org/x/sys/unix/ztypes_linux_mips.go

@@ -182,6 +182,55 @@ type FscryptKey struct {
 	Size uint32
 }
 
+type FscryptPolicyV1 struct {
+	Version                   uint8
+	Contents_encryption_mode  uint8
+	Filenames_encryption_mode uint8
+	Flags                     uint8
+	Master_key_descriptor     [8]uint8
+}
+
+type FscryptPolicyV2 struct {
+	Version                   uint8
+	Contents_encryption_mode  uint8
+	Filenames_encryption_mode uint8
+	Flags                     uint8
+	_                         [4]uint8
+	Master_key_identifier     [16]uint8
+}
+
+type FscryptGetPolicyExArg struct {
+	Size   uint64
+	Policy [24]byte
+}
+
+type FscryptKeySpecifier struct {
+	Type uint32
+	_    uint32
+	U    [32]byte
+}
+
+type FscryptAddKeyArg struct {
+	Key_spec FscryptKeySpecifier
+	Raw_size uint32
+	_        [9]uint32
+}
+
+type FscryptRemoveKeyArg struct {
+	Key_spec             FscryptKeySpecifier
+	Removal_status_flags uint32
+	_                    [5]uint32
+}
+
+type FscryptGetKeyStatusArg struct {
+	Key_spec     FscryptKeySpecifier
+	_            [6]uint32
+	Status       uint32
+	Status_flags uint32
+	User_count   uint32
+	_            [13]uint32
+}
+
 type KeyctlDHParams struct {
 	Private int32
 	Prime   int32
@@ -259,7 +308,7 @@ type RawSockaddrRFCOMM struct {
 type RawSockaddrCAN struct {
 	Family  uint16
 	Ifindex int32
-	Addr    [8]byte
+	Addr    [16]byte
 }
 
 type RawSockaddrALG struct {
@@ -430,7 +479,7 @@ const (
 	SizeofSockaddrHCI       = 0x6
 	SizeofSockaddrL2        = 0xe
 	SizeofSockaddrRFCOMM    = 0xa
-	SizeofSockaddrCAN       = 0x10
+	SizeofSockaddrCAN       = 0x18
 	SizeofSockaddrALG       = 0x58
 	SizeofSockaddrVM        = 0x10
 	SizeofSockaddrXDP       = 0x10
@@ -2038,6 +2087,7 @@ type XDPRingOffset struct {
 	Producer uint64
 	Consumer uint64
 	Desc     uint64
+	Flags    uint64
 }
 
 type XDPMmapOffsets struct {
@@ -2052,6 +2102,8 @@ type XDPUmemReg struct {
 	Len      uint64
 	Size     uint32
 	Headroom uint32
+	Flags    uint32
+	_        [4]byte
 }
 
 type XDPStatistics struct {
@@ -2660,7 +2712,7 @@ const (
 	DEVLINK_CMD_DPIPE_ENTRIES_GET             = 0x20
 	DEVLINK_CMD_DPIPE_HEADERS_GET             = 0x21
 	DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET      = 0x22
-	DEVLINK_CMD_MAX                           = 0x3c
+	DEVLINK_CMD_MAX                           = 0x44
 	DEVLINK_PORT_TYPE_NOTSET                  = 0x0
 	DEVLINK_PORT_TYPE_AUTO                    = 0x1
 	DEVLINK_PORT_TYPE_ETH                     = 0x2
@@ -2740,7 +2792,7 @@ const (
 	DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE     = 0x3c
 	DEVLINK_ATTR_PAD                          = 0x3d
 	DEVLINK_ATTR_ESWITCH_ENCAP_MODE           = 0x3e
-	DEVLINK_ATTR_MAX                          = 0x80
+	DEVLINK_ATTR_MAX                          = 0x89
 	DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE     = 0x0
 	DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX  = 0x1
 	DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT      = 0x0