s3: add Abrha Storage provider
This commit is contained in:
javad seydi authored and GitHub committed
1 parent
82cb84b201
commit
d459c3f992
8 files changed
+170
No files matched your search
@@ -24,6 +24,7 @@ directories to and from different cloud storage providers.
|
||||
## Storage providers
|
||||
|
||||
- 1Fichier [:page_facing_up:](https://rclone.org/fichier/)
|
||||
- Abrha Storage [:page_facing_up:](https://rclone.org/s3/#abrhastorage)
|
||||
- Akamai Netstorage [:page_facing_up:](https://rclone.org/netstorage/)
|
||||
- Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
|
||||
- Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
|
||||
|
||||
@@ -161,6 +161,7 @@ In `backend/s3/provider/YourProvider.yaml`
|
||||
type Quirks struct {
|
||||
ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2
|
||||
ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style
|
||||
NoCheckBucket *bool `yaml:"no_check_bucket,omitempty"` // true = skip HeadBucket/CreateBucket
|
||||
ListURLEncode *bool `yaml:"list_url_encode,omitempty"`
|
||||
UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"`
|
||||
UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"`
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
name: AbrhaStorage
|
||||
description: Abrha Storage (ParsPack)
|
||||
region: {}
|
||||
endpoint: {}
|
||||
quirks:
|
||||
list_version: 1
|
||||
force_path_style: true # per-bucket hosts, e.g. https://c123456.parspack.net
|
||||
use_multipart_etag: false
|
||||
# ParsPack documents --s3-no-check-bucket for bucket check access errors.
|
||||
# https://parspack.com/blog/hosting/connecting-to-s3-bucket-with-cloud-storage
|
||||
no_check_bucket: true
|
||||
@@ -22,6 +22,7 @@ var NewYamlMap = orderedmap.New[string, string]
|
||||
type Quirks struct {
|
||||
ListVersion *int `yaml:"list_version,omitempty"` // 1 or 2
|
||||
ForcePathStyle *bool `yaml:"force_path_style,omitempty"` // true = path-style
|
||||
NoCheckBucket *bool `yaml:"no_check_bucket,omitempty"` // true = skip HeadBucket/CreateBucket
|
||||
ListURLEncode *bool `yaml:"list_url_encode,omitempty"`
|
||||
UseMultipartEtag *bool `yaml:"use_multipart_etag,omitempty"`
|
||||
UseAlreadyExists *bool `yaml:"use_already_exists,omitempty"`
|
||||
|
||||
@@ -470,6 +470,8 @@ rclone does if you know the bucket exists already.
|
||||
It can also be needed if the user you are using does not have bucket
|
||||
creation permissions. Before v1.52.0 this would have passed silently
|
||||
due to a bug.
|
||||
|
||||
Some providers (e.g. Abrha Storage) set this automatically.
|
||||
`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
@@ -1747,6 +1749,12 @@ func setQuirks(opt *Options, provider *Provider) {
|
||||
opt.ForcePathStyle = false
|
||||
}
|
||||
|
||||
// Some providers reject HeadBucket and CreateBucket for keys that can
|
||||
// still read and write objects. Skip that check when required.
|
||||
if provider.Quirks.NoCheckBucket != nil && *provider.Quirks.NoCheckBucket {
|
||||
opt.NoCheckBucket = true
|
||||
}
|
||||
|
||||
// Set the correct list version if not manually set
|
||||
if opt.ListVersion == 0 {
|
||||
if provider.Quirks.ListVersion != nil {
|
||||
|
||||
@@ -973,3 +973,13 @@ func TestBufferForObjectLockMD5(t *testing.T) {
|
||||
assert.Equal(t, inUse, pool.Global().InUse(), "pool buffers leaked")
|
||||
})
|
||||
}
|
||||
|
||||
func TestSetQuirksNoCheckBucket(t *testing.T) {
|
||||
opt := &Options{}
|
||||
setQuirks(opt, loadProvider("AbrhaStorage"))
|
||||
assert.True(t, opt.NoCheckBucket)
|
||||
|
||||
opt = &Options{}
|
||||
setQuirks(opt, loadProvider("AWS"))
|
||||
assert.False(t, opt.NoCheckBucket)
|
||||
}
|
||||
@@ -113,6 +113,7 @@ WebDAV or S3, that work out of the box.)
|
||||
|
||||
{{< provider_list >}}
|
||||
{{< provider name="1Fichier" home="https://1fichier.com/" config="/fichier/" start="true">}}
|
||||
{{< provider name="Abrha Storage" home="https://parspack.com/cloud-storage" config="/s3/#abrhastorage" >}}
|
||||
{{< provider name="Akamai Netstorage" home="https://www.akamai.com/us/en/products/media-delivery/netstorage.jsp" config="/netstorage/" >}}
|
||||
{{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
|
||||
{{< provider name="Amazon S3" home="https://aws.amazon.com/s3/" config="/s3/" >}}
|
||||
|
||||
@@ -13,6 +13,7 @@ The S3 backend can be used with a number of different providers:
|
||||
|
||||
{{< provider_list >}}
|
||||
{{< provider name="AWS S3" home="https://aws.amazon.com/s3/" config="/s3/#configuration" start="true" >}}
|
||||
{{< provider name="Abrha Storage" home="https://parspack.com/cloud-storage" config="/s3/#abrhastorage" >}}
|
||||
{{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
|
||||
{{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
|
||||
{{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}}
|
||||
@@ -5095,6 +5096,142 @@ endpoint = http://[IP of Snowball]:8080
|
||||
upload_cutoff = 0
|
||||
```
|
||||
|
||||
### Abrha Storage {#abrhastorage}
|
||||
|
||||
[Abrha Storage](https://parspack.com/cloud-storage) is S3-compatible object
|
||||
storage. Each space has its own endpoint, shown in the storage panel, of the
|
||||
form `https://c123456.parspack.net`. The bucket name in rclone paths is that
|
||||
same id (`c123456`).
|
||||
|
||||
Connection details are in the ParsPack S3 guide:
|
||||
|
||||
<https://parspack.com/blog/hosting/connecting-to-s3-bucket-with-cloud-storage>
|
||||
|
||||
Leave the region empty. The provider uses path-style access and automatically
|
||||
skips the bucket existence check, matching ParsPack's documented workaround
|
||||
for bucket check access errors. Create the bucket in the panel before use.
|
||||
|
||||
Here is an example of making a remote. First run:
|
||||
|
||||
```console
|
||||
rclone config
|
||||
```
|
||||
|
||||
This will guide you through an interactive setup process.
|
||||
|
||||
```text
|
||||
No remotes found, make a new one?
|
||||
n) New remote
|
||||
s) Set configuration password
|
||||
q) Quit config
|
||||
n/s/q> n
|
||||
|
||||
Enter name for new remote.
|
||||
name> pspk
|
||||
|
||||
Option Storage.
|
||||
Type of storage to configure.
|
||||
Choose a number from below, or type in your own value.
|
||||
[snip]
|
||||
XX / Amazon S3 Compliant Storage Providers including AWS, AbrhaStorage, ...
|
||||
\ (s3)
|
||||
[snip]
|
||||
Storage> s3
|
||||
|
||||
Option provider.
|
||||
Choose your S3 provider.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
[snip]
|
||||
XX / Abrha Storage (ParsPack)
|
||||
\ (AbrhaStorage)
|
||||
[snip]
|
||||
provider> AbrhaStorage
|
||||
|
||||
Option env_auth.
|
||||
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
|
||||
Only applies if access_key_id and secret_access_key is blank.
|
||||
Choose a number from below, or type in your own boolean value (true or false).
|
||||
Press Enter for the default (false).
|
||||
1 / Enter AWS credentials in the next step.
|
||||
\ (false)
|
||||
2 / Get AWS credentials from the environment (env vars or IAM).
|
||||
\ (true)
|
||||
env_auth> 1
|
||||
|
||||
Option access_key_id.
|
||||
AWS Access Key ID.
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
access_key_id> YOURACCESSKEY
|
||||
|
||||
Option secret_access_key.
|
||||
AWS Secret Access Key (password).
|
||||
Leave blank for anonymous access or runtime credentials.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
secret_access_key> YOURSECRETACCESSKEY
|
||||
|
||||
Option region.
|
||||
Region to connect to.
|
||||
Leave blank if you are using an S3 clone and you don't have a region.
|
||||
Choose a number from below, or type in your own value.
|
||||
Press Enter to leave empty.
|
||||
/ Use this if unsure.
|
||||
1 | Will use v4 signatures and an empty region.
|
||||
\ ()
|
||||
/ Use this only if v4 signatures don't work.
|
||||
2 | E.g. pre Jewel/v10 CEPH.
|
||||
\ (other-v2-signature)
|
||||
region>
|
||||
|
||||
Option endpoint.
|
||||
Endpoint for S3 API.
|
||||
Required when using an S3 clone.
|
||||
Enter a value. Press Enter to leave empty.
|
||||
endpoint> https://c123456.parspack.net
|
||||
|
||||
Option bucket_object_lock_enabled.
|
||||
Enable Object Lock when creating new buckets.
|
||||
Enter a boolean value (true or false). Press Enter for the default (false).
|
||||
bucket_object_lock_enabled>
|
||||
|
||||
Edit advanced config?
|
||||
y) Yes
|
||||
n) No (default)
|
||||
y/n> n
|
||||
|
||||
Configuration complete.
|
||||
Options:
|
||||
- type: s3
|
||||
- provider: AbrhaStorage
|
||||
- access_key_id: YOURACCESSKEY
|
||||
- secret_access_key: YOURSECRETACCESSKEY
|
||||
- endpoint: https://c123456.parspack.net
|
||||
Keep this "pspk" remote?
|
||||
y) Yes this is OK (default)
|
||||
e) Edit this remote
|
||||
d) Delete this remote
|
||||
y/e/d> y
|
||||
```
|
||||
|
||||
This will leave the config file looking like this.
|
||||
|
||||
```ini
|
||||
[pspk]
|
||||
type = s3
|
||||
provider = AbrhaStorage
|
||||
access_key_id = YOURACCESSKEY
|
||||
secret_access_key = YOURSECRETACCESSKEY
|
||||
endpoint = https://c123456.parspack.net
|
||||
```
|
||||
|
||||
`force_path_style` and `no_check_bucket` are applied automatically for
|
||||
this provider, so they do not need to be set. `no_check_bucket` can't be
|
||||
overridden with `no_check_bucket = false`, since buckets can only be
|
||||
created in the ParsPack panel.
|
||||
|
||||
Paths are `remote:bucket`, for example `pspk:c123456`.
|
||||
|
||||
### Alibaba OSS {#alibaba-oss}
|
||||
|
||||
Here is an example of making an [Alibaba Cloud (Aliyun) OSS](https://www.alibabacloud.com/product/oss/)
|
||||
|
||||
Reference in new issue
Block a user