rclone 工具使用及问题汇总

1. 命令行方式传输文件, 比如 copy, sync, move 等子命令;
2. mount 分区挂载方式传输文件, 类似挂载 nfs 等方式到本地;

Fatal error: failed to mount FUSE fs: kernel FUSE version is too old: 7.14 < 7.17

grep FUSE_KERNEL_MINOR_VERSION /usr/include/linux/fuse.h
#define FUSE_KERNEL_MINOR_VERSION 14

 rclone --config /etc/rclone/rclone.conf sync /tmp/file gcp_oss:test1_gcs/
 rclone --config /etc/rclone/rclone.conf ls gcp_oss:test1_gcs/

rsync: failed to set time on ....: Input/output error  -- 不要用 -a 选项, 对象存储不支持修改 time 等参数.
rsync: rename ... -> ....: Input/output error

mkdir /export/rsync_temp
rsync -vZ -T /export/rsync_temp source dest

1. rsync 模块以 root 用户启动, 禁止 chroot;
2. rsync 模块 path 指定为挂载点的上层目录, 比如云盘挂载到本地 /mnt/rclone_mount/test1_gcs, 就将 path 指定为 /mnt/rclone_mount;
3. 在 path 目录创建 rsync 的临时目录， 比如 /mnt/rclone_mount/rsync_tmp;
4. 其它主机通过以下方式传输, 假定 rsync 模块名为 rsync_test1, 用户为 rsync_user, 云存储桶名为 test1_gcs, rsync 不要指定 -a 选项:
   rsync -T /rsync_tmp --password-file=/tmp/rsync.pass  -v --progress  /tmp/rsync.log  rsync_user@10.1.1.2::rsync_test1/test1_gcs/

# 创建服务
sc create rclone binpath= "D:\rclone-windows\rclone.exe mount --config D:\rclone-windows\rclone.conf --allow-non-empty --allow-other --vfs-cache-mode full --dir-cache-time 3m --poll-interval 1m --vfs-read-chunk-size-limit 128M --vfs-write-back 0s --log-level INFO --log-file D:\rclone-windows\log\rclone-sftp.log -o FileSecurity=\"O:WDG:WDD:NO_ACCESS_CONTROL\" --bwlimit 100M rclone_mount:/mnt/dl_tmp D:\rclone_mount" displayname= "Rclone Service" depend= Tcpip start= auto

# 删除服务
sc delete rclone

// rclone flags
--cache-dir string                     Directory rclone will use for caching (default "/root/.cache/rclone")
...
--fs-cache-expire-duration duration    Cache remotes for this long (0 to disable caching) (default 5m0s)

rclone --cache-dir /export/rclone/tmp  --fs-cache-expire-duration 2m mount ....

[Unit]
Description=rclone: Remote FUSE filesystem for cloud storage config %i
Documentation=man:rclone(1)
After=network-online.target
Wants=network-online.target
AssertPathIsDirectory=/mnt/rclone_mount/

[Service]
Type=notify
ExecStartPre=/bin/mkdir -p /mnt/rclone_mount/%i
ExecStart=/usr/bin/rclone mount --config /etc/rclone/rclone.conf --allow-non-empty --allow-other --vfs-cache-mode full --vfs-cache-max-size 500M --dir-cache-time 5m --poll-interval 1m --vfs-read-chunk-size-limit 128M --vfs-read-chunk-size-limit off --log-level INFO --log-file /var/log/rclone/rclone-%i.log --bwlimit 20M --umask 022 rclone_oss_consul:%i /mnt/rclone_mount/%i
ExecStop=/bin/fusermount -uz /mnt/rclone_mount/%i
Restart=always
RestartSec=10

[Install]
WantedBy=default.target

systemctl enable rclone@bucketname
systemctl start rclone@bucketname
systemctl status rclone@bucketname   -->  查看服务状态

1. 一个账号或 key 可以管理所有桶;
2. 一个账号或 key 映射为多个 consul key, 每个 key 可以对应单个项目;
3. 不可见账号或 key, 所有数据的传输均以 consul key 的形式通信;
4. 权限管理简单, 使用者和对象存储管理者均简单方便;

       client
   +-------------+          https         +---------------+
   |   rclone    |      --------------->  | consul server |
   +-------------+                        +---------------+
 
1. rclone 从 consul server 获取以 service-consul-key 为键的信息 value;
2. rclone 解析 value 中的 service_account 字段信息, 并进行解密;
3. 将解密内容直接赋值给选项 gcs-client-secret;
4. rclone 校验 value 的 buckets 选项, 通过后开始挂载或者传输数据;

--gcs-service-consul-key
--gcs-service-consul-server
--gcs-service-consul-token
--gcs-service-consul-tls

service_consul_server = 10.1.1.2:8500
service_consul_token = ck843784-1fe8-95c9-12df-06ac368528gf
service_consul_key = 40f51886b53b4966c9d009e2ba05efb7
service_consul_tls = true

--gcs-service-account-file
--gcs-client-secret

--s3-access-key-id
--s3-secret-access-key

# gcs 示例
key:   oss/gcs/2d1127ff54d14cecb291cdba4460d890
value: {
  "team": "myteam",
  "service_account": "<aesutils 执行后加密的内容>",
  "buckets": ["test1", "test2", "test3"]
}

# s3 示例
key:   oss/s3/2c1127ff64d14cebb291cdba4460c809
value: {
  "team": "myteam",
  "provider": "Kingsoft oss",
  "access_key_id": "<aesutils 对 access key 加密后的内容>",
  "secret_access_key": "aesutils 对 access secret 加密后的内容",
  "buckets": ["test1", "test2", "test3"]
}

gcp      - gcs
aws      - s3
aliyun   - oss
tencent  - cos
kingsoft - ks3

[rclone_oss_consul]
type = google cloud storage
object_acl = bucketOwnerFullControl
bucket_acl = private
auth_url = https://accounts.google.com/o/oauth2/auth
token_url = https://oauth2.googleapis.com/token
service_consul_server = <consul server>
service_consul_token = <consul token>
service_consul_key = <consul key>
service_consul_tls = true

[rclone_oss_consul]
type = s3
provider = AWS
env_auth = false
service_consul_server = <consul server>
service_consul_token = <sonsul token>
service_consul_key = <consul key>
service_consul_tls = true
region = <region>
endpoint = <endpoint>
acl = private
bucket_acl = private
storage_class = STANDARD

[rclone_oss_consul]
type = s3
provider = Alibaba
env_auth = false
service_consul_server = <consul server>
service_consul_token = <sonsul token>
service_consul_key = <consul key>
service_consul_tls = true
endpoint = <endpoint>
acl = private
bucket_acl = private
storage_class = STANDARD

[rclone_oss_consul]
type = s3
provider = TencentCOS
env_auth = false
service_consul_server = <consul server>
service_consul_token = <sonsul token>
service_consul_key = <consul key>
service_consul_tls = true
endpoint = <endpoint>
acl = private
bucket_acl = private
storage_class = STANDARD

[rclone_oss_consul]
type = s3
provider = Other
env_auth = false
service_consul_server = <consul server>
service_consul_token = <sonsul token>
service_consul_key = <consul key>
service_consul_tls = true
endpoint = <endpoint>
region = <region>                      ---  金山云必须要指定 region
acl = private
bucket_acl = private
storage_class = STANDARD

1. 尽量保证一个项目只有一两台机器运行 rclone, 不要过多的挂载, 其它机器可以通过 rsync 传输文件;
2. rclone 做好限制, 比如 20M, 不要设置太大避免占用很多带宽;
3. rclone 的 mount 方式, 仅在启动的时候和 consul 通信, 挂载后不会再有交互;
4. rclone 的命令行方式, 执行一次就会和 consul 通信一次, 如果网络质量不好, 建议采用 mount 方式和 consul 通信;