leftyer/JuiceFS
1
0
Fork 0
mirror of https://gitee.com/juicedata/JuiceFS.git synced 2025-12-06 09:39:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

add juicefs.skip-dir-mtime for sdk (#4651)

Browse Source
This commit is contained in:
tangyoupeng
2024-04-07 15:49:31 +08:00
committed by GitHub
parent 0681c7b300
commit b5efe8a0ff
11 changed files with 74 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
cmd/flags.go
View File

@@ -18,14 +18,10 @@ package cmd
import (
"fmt"
"github.com/urfave/cli/v2"
"os"
"path"
"runtime"
"strconv"
"strings"
"time"
"github.com/urfave/cli/v2"
)
func globalFlags() []cli.Flag {
@@ -350,29 +346,3 @@ func expandFlags(compoundFlags ...[]cli.Flag) []cli.Flag {
}
return flags
}
func duration(s string) time.Duration {
if s == "" {
return 0
}
v, err := strconv.ParseFloat(s, 64)
if err == nil {
return time.Microsecond * time.Duration(v*1e6)
}
err = nil
var d time.Duration
p := strings.Index(s, "d")
if p >= 0 {
v, err = strconv.ParseFloat(s[:p], 64)
}
if err == nil && s[p+1:] != "" {
d, err = time.ParseDuration(s[p+1:])
}
if err != nil {
logger.Warnf("Invalid duration value: %s, setting it to 0", s)
return 0
}
return d + time.Hour*time.Duration(v*24)
}

3
cmd/flags_test.go
View File

@@ -1,6 +1,7 @@
package cmd
import (
"github.com/juicedata/juicefs/pkg/utils"
"testing"
"time"
@@ -49,7 +50,7 @@ func Test_duration(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, duration(tt.args.s), "duration(%v)", tt.args.s)
assert.Equalf(t, tt.want, utils.Duration(tt.args.s), "duration(%v)", tt.args.s)
})
}
}

7
cmd/gateway.go
View File

@@ -22,6 +22,7 @@ package cmd
import (
"context"
"errors"
"github.com/juicedata/juicefs/pkg/utils"
_ "net/http/pprof"
"os"
"os/signal"
@@ -249,9 +250,9 @@ func initForSvc(c *cli.Context, mp string, metaUrl string) (*vfs.Config, *fs.Fil
}()
vfsConf := getVfsConf(c, metaConf, format, chunkConf)
vfsConf.AccessLog = c.String("access-log")
vfsConf.AttrTimeout = duration(c.String("attr-cache"))
vfsConf.EntryTimeout = duration(c.String("entry-cache"))
vfsConf.DirEntryTimeout = duration(c.String("dir-entry-cache"))
vfsConf.AttrTimeout = utils.Duration(c.String("attr-cache"))
vfsConf.EntryTimeout = utils.Duration(c.String("entry-cache"))
vfsConf.DirEntryTimeout = utils.Duration(c.String("dir-entry-cache"))
initBackgroundTasks(c, vfsConf, metaConf, metaCli, blob, registerer, registry)
jfs, err := fs.NewFileSystem(vfsConf, metaCli, store)

6
cmd/mdtest.go
View File

@@ -217,9 +217,9 @@ func initForMdtest(c *cli.Context, mp string, metaUrl string) *fs.FileSystem {
conf := getVfsConf(c, metaConf, format, chunkConf)
conf.AccessLog = c.String("access-log")
conf.AttrTimeout = duration(c.String("attr-cache"))
conf.EntryTimeout = duration(c.String("entry-cache"))
conf.DirEntryTimeout = duration(c.String("dir-entry-cache"))
conf.AttrTimeout = utils.Duration(c.String("attr-cache"))
conf.EntryTimeout = utils.Duration(c.String("entry-cache"))
conf.DirEntryTimeout = utils.Duration(c.String("dir-entry-cache"))
metricsAddr := exposeMetrics(c, registerer, registry)
m.InitMetrics(registerer)

16
cmd/mount.go
View File

@@ -254,7 +254,7 @@ func getVfsConf(c *cli.Context, metaConf *meta.Config, format *meta.Format, chun
Format: *format,
Version: version.Version(),
Chunk: chunkConf,
BackupMeta: duration(c.String("backup-meta")),
BackupMeta: utils.Duration(c.String("backup-meta")),
BackupSkipTrash: c.Bool("backup-skip-trash"),
Port: &vfs.Port{DebugAgent: debugAgent, PyroscopeAddr: c.String("pyroscope")},
PrefixInternal: c.Bool("prefix-internal"),
@@ -294,10 +294,10 @@ func getMetaConf(c *cli.Context, mp string, readOnly bool) *meta.Config {
conf.NoBGJob = c.Bool("no-bgjob")
conf.OpenCache = duration(c.String("open-cache"))
conf.OpenCacheLimit = c.Uint64("open-cache-limit")
conf.Heartbeat = duration(c.String("heartbeat"))
conf.Heartbeat = utils.Duration(c.String("heartbeat"))
conf.MountPoint = mp
conf.Subdir = c.String("subdir")
conf.SkipDirMtime = duration(c.String("skip-dir-mtime"))
conf.SkipDirMtime = utils.Duration(c.String("skip-dir-mtime"))
conf.Sid, _ = strconv.ParseUint(os.Getenv("_JFS_META_SID"), 10, 64)
atimeMode := c.String("atime-mode")
@@ -320,8 +320,8 @@ func getChunkConf(c *cli.Context, format *meta.Format) *chunk.Config {
Compress: format.Compression,
HashPrefix: format.HashPrefix,
GetTimeout: duration(c.String("get-timeout")),
PutTimeout: duration(c.String("put-timeout")),
GetTimeout: utils.Duration(c.String("get-timeout")),
PutTimeout: utils.Duration(c.String("put-timeout")),
MaxUpload: c.Int("max-uploads"),
MaxRetries: c.Int("io-retries"),
Writeback: c.Bool("writeback"),
@@ -329,7 +329,7 @@ func getChunkConf(c *cli.Context, format *meta.Format) *chunk.Config {
BufferSize: utils.ParseBytes(c, "buffer-size", 'M'),
UploadLimit: utils.ParseMbps(c, "upload-limit") * 1e6 / 8,
DownloadLimit: utils.ParseMbps(c, "download-limit") * 1e6 / 8,
UploadDelay: duration(c.String("upload-delay")),
UploadDelay: utils.Duration(c.String("upload-delay")),
UploadHours: c.String("upload-hours"),
CacheDir: c.String("cache-dir"),
@@ -339,8 +339,8 @@ func getChunkConf(c *cli.Context, format *meta.Format) *chunk.Config {
CacheFullBlock: !c.Bool("cache-partial-only"),
CacheChecksum: c.String("verify-cache-checksum"),
CacheEviction: c.String("cache-eviction"),
CacheScanInterval: duration(c.String("cache-scan-interval")),
CacheExpire: duration(c.String("cache-expire")),
CacheScanInterval: utils.Duration(c.String("cache-scan-interval")),
CacheExpire: utils.Duration(c.String("cache-expire")),
AutoCreate: true,
}
if chunkConf.UploadLimit == 0 {

6
cmd/mount_unix.go
View File

@@ -724,9 +724,9 @@ func mountMain(v *vfs.VFS, c *cli.Context) {
disableUpdatedb()
}
conf := v.Conf
conf.AttrTimeout = duration(c.String("attr-cache"))
conf.EntryTimeout = duration(c.String("entry-cache"))
conf.DirEntryTimeout = duration(c.String("dir-entry-cache"))
conf.AttrTimeout = utils.Duration(c.String("attr-cache"))
conf.EntryTimeout = utils.Duration(c.String("entry-cache"))
conf.DirEntryTimeout = utils.Duration(c.String("dir-entry-cache"))
conf.NonDefaultPermission = c.Bool("non-default-permission")
rootSquash := c.String("root-squash")
if rootSquash != "" {

41
docs/en/deployment/hadoop_java_sdk.md
View File

@@ -190,27 +190,28 @@ Please refer to the following table to set the relevant parameters of the JuiceF
#### Other Configurations
| Configuration | Default Value | Description |
|---------------------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `juicefs.bucket` | | Specify a different endpoint for object storage |
| `juicefs.debug` | `false` | Whether enable debug log |
| `juicefs.access-log` | | Access log path. Ensure Hadoop application has write permission, e.g. `/tmp/juicefs.access.log`. The log file will rotate automatically to keep at most 7 files. |
| `juicefs.superuser` | `hdfs` | The super user |
| `juicefs.supergroup` | `supergroup` | The super user group |
| `juicefs.users` | `null` | The path of username and UID list file, e.g. `jfs://name/etc/users`. The file format is `<username>:<UID>`, one user per line. |
| `juicefs.groups` | `null` | The path of group name, GID and group members list file, e.g. `jfs://name/etc/groups`. The file format is `<group-name>:<GID>:<username1>,<username2>`, one group per line. |
| `juicefs.umask` | `null` | The umask used when creating files and directories (e.g. `0022`), default value is `fs.permissions.umask-mode`. |
| `juicefs.push-gateway` | | [Prometheus Pushgateway](https://github.com/prometheus/pushgateway) address, format is `<host>:<port>`. |
| `juicefs.push-auth` | | [Prometheus basic auth](https://prometheus.io/docs/guides/basic-auth) information, format is `<username>:<password>`. |
| `juicefs.push-graphite` | | [Graphite](https://graphiteapp.org) address, format is `<host>:<port>`. |
| `juicefs.push-interval` | 10 | Metric push interval (in seconds) |
| `juicefs.push-labels` | | Metric labels, format is `key1:value1;key2:value2`. |
| `juicefs.fast-resolve` | `true` | Whether enable faster metadata lookup using Redis Lua script |
| Configuration | Default Value | Description |
|-------------------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `juicefs.bucket` | | Specify a different endpoint for object storage |
| `juicefs.debug` | `false` | Whether enable debug log |
| `juicefs.access-log` | | Access log path. Ensure Hadoop application has write permission, e.g. `/tmp/juicefs.access.log`. The log file will rotate automatically to keep at most 7 files. |
| `juicefs.superuser` | `hdfs` | The super user |
| `juicefs.supergroup` | `supergroup` | The super user group |
| `juicefs.users` | `null` | The path of username and UID list file, e.g. `jfs://name/etc/users`. The file format is `<username>:<UID>`, one user per line. |
| `juicefs.groups` | `null` | The path of group name, GID and group members list file, e.g. `jfs://name/etc/groups`. The file format is `<group-name>:<GID>:<username1>,<username2>`, one group per line. |
| `juicefs.umask` | `null` | The umask used when creating files and directories (e.g. `0022`), default value is `fs.permissions.umask-mode`. |
| `juicefs.push-gateway` | | [Prometheus Pushgateway](https://github.com/prometheus/pushgateway) address, format is `<host>:<port>`. |
| `juicefs.push-auth` | | [Prometheus basic auth](https://prometheus.io/docs/guides/basic-auth) information, format is `<username>:<password>`. |
| `juicefs.push-graphite` | | [Graphite](https://graphiteapp.org) address, format is `<host>:<port>`. |
| `juicefs.push-interval` | 10 | Metric push interval (in seconds) |
| `juicefs.push-labels` | | Metric labels, format is `key1:value1;key2:value2`. |
| `juicefs.fast-resolve` | `true` | Whether enable faster metadata lookup using Redis Lua script |
| `juicefs.no-usage-report` | `false` | Whether disable usage reporting. JuiceFS only collects anonymous usage data (e.g. version number), no user or any sensitive data will be collected. |
| `juicefs.no-bgjob` | `false` | Disable background jobs (clean-up, backup, etc.) |
| `juicefs.backup-meta` | 3600 | Interval (in seconds) to automatically backup metadata in the object storage (0 means disable backup) |
|`juicefs.backup-skip-trash`| `false` | Skip files and directories in trash when backup metadata. |
| `juicefs.heartbeat` | 12 | Heartbeat interval (in seconds) between client and metadata engine. It's recommended that all clients use the same value. |
| `juicefs.no-bgjob` | `false` | Disable background jobs (clean-up, backup, etc.) |
| `juicefs.backup-meta` | 3600 | Interval (in seconds) to automatically backup metadata in the object storage (0 means disable backup) |
| `juicefs.backup-skip-trash` | `false` | Skip files and directories in trash when backup metadata. |
| `juicefs.heartbeat` | 12 | Heartbeat interval (in seconds) between client and metadata engine. It's recommended that all clients use the same value. |
| `juicefs.skip-dir-mtime` | 100ms | Minimal duration to modify parent dir mtime. |
#### Multiple file systems configuration

5
docs/zh_cn/deployment/hadoop_java_sdk.md
View File

@@ -204,15 +204,16 @@ make win
| `juicefs.push-auth` | | [Prometheus 基本认证](https://prometheus.io/docs/guides/basic-auth)信息，格式为 `<username>:<password>`。 |
| `juicefs.push-graphite` | | [Graphite](https://graphiteapp.org) 地址，格式为 `<host>:<port>`。 |
| `juicefs.push-interval` | 10 | 指标推送的时间间隔，单位为秒。 |
| `juicefs.push-labels` | | 指标额外标签，格式为 `key1:value1;key2:value2`。 |
| `juicefs.push-labels` | | 指标额外标签，格式为 `key1:value1;key2:value2`。 |
| `juicefs.fast-resolve` | `true` | 是否开启快速元数据查找（通过 Redis Lua 脚本实现） |
| `juicefs.no-usage-report` | `false` | 是否上报数据。仅上版本号等使用量数据，不包含任何用户信息。 |
| `juicefs.block.size` | `134217728` | 单位为字节，同 HDFS 的 `dfs.blocksize`，默认 128 MB |
| `juicefs.file.checksum` | `false` | DistCp 使用 `-update` 参数时，是否计算文件 Checksum |
| `juicefs.no-bgjob` | `false` | 是否关闭后台任务（清理、备份等） |
| `juicefs.backup-meta` | 3600 | 自动将 JuiceFS 元数据备份到对象存储间隔（单位：秒），设置为 0 关闭自动备份 |
|`juicefs.backup-skip-trash`| `false` | 备份元数据时忽略回收站中的文件和目录。 |
|`juicefs.backup-skip-trash`| `false` | 备份元数据时忽略回收站中的文件和目录。 |
| `juicefs.heartbeat` | 12 | 客户端和元数据引擎之间的心跳间隔（单位：秒），建议所有客户端都设置一样 |
| `juicefs.skip-dir-mtime` | 100ms | 修改父目录 mtime 间隔。 |
#### 多文件系统配置

26
pkg/utils/utils.go
View File

@@ -247,3 +247,29 @@ func LookupGroup(name string) int {
groups[name] = gid
return gid
}
func Duration(s string) time.Duration {
if s == "" {
return 0
}
v, err := strconv.ParseFloat(s, 64)
if err == nil {
return time.Microsecond * time.Duration(v*1e6)
}
err = nil
var d time.Duration
p := strings.Index(s, "d")
if p >= 0 {
v, err = strconv.ParseFloat(s[:p], 64)
}
if err == nil && s[p+1:] != "" {
d, err = time.ParseDuration(s[p+1:])
}
if err != nil {
logger.Warnf("Invalid duration value: %s, setting it to 0", s)
return 0
}
return d + time.Hour*time.Duration(v*24)
}

2
sdk/java/libjfs/main.go
View File

@@ -300,6 +300,7 @@ type javaConf struct {
MaxUploads int `json:"maxUploads"`
MaxDeletes int `json:"maxDeletes"`
SkipDirNlink int `json:"skipDirNlink"`
SkipDirMtime string `json:"skipDirMtime"`
IORetries int `json:"ioRetries"`
GetTimeout int `json:"getTimeout"`
PutTimeout int `json:"putTimeout"`
@@ -441,6 +442,7 @@ func jfs_init(cname, jsonConf, user, group, superuser, supergroup *C.char) int64
metaConf.Retries = jConf.IORetries
metaConf.MaxDeletes = jConf.MaxDeletes
metaConf.SkipDirNlink = jConf.SkipDirNlink
metaConf.SkipDirMtime = utils.Duration(jConf.SkipDirMtime)
metaConf.ReadOnly = jConf.ReadOnly
metaConf.NoBGJob = jConf.NoBGJob || jConf.NoSession
metaConf.OpenCache = time.Duration(jConf.OpenCache * 1e9)

1
sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java
View File

@@ -390,6 +390,7 @@ public class JuiceFileSystemImpl extends FileSystem {
obj.put("maxUploads", Integer.valueOf(getConf(conf, "max-uploads", "20")));
obj.put("maxDeletes", Integer.valueOf(getConf(conf, "max-deletes", "10")));
obj.put("skipDirNlink", Integer.valueOf(getConf(conf, "skip-dir-nlink", "20")));
obj.put("skipDirMtime", getConf(conf, "skip-dir-mtime", "100ms"));
obj.put("uploadLimit", Integer.valueOf(getConf(conf, "upload-limit", "0")));
obj.put("downloadLimit", Integer.valueOf(getConf(conf, "download-limit", "0")));
obj.put("ioRetries", Integer.valueOf(getConf(conf, "io-retries", "10")));