mirror of
https://gitee.com/juicedata/JuiceFS.git
synced 2025-12-06 09:39:14 +08:00
pkg/object: support cifs/smb (#6368)
Signed-off-by: Xuhui zhang <xuhui@juicedata.io>
This commit is contained in:
1
.github/scripts/prepare_db.sh
vendored
1
.github/scripts/prepare_db.sh
vendored
@@ -118,6 +118,7 @@ prepare_db() {
|
||||
install_gluster
|
||||
install_webdav
|
||||
docker run -d --name sftp -p 2222:22 juicedata/ci-sftp
|
||||
docker run -d --name samba -p 4445:445 -e "USER=samba" -e "PASS=secret" dockurr/samba
|
||||
install_etcd
|
||||
.github/scripts/setup-hdfs.sh
|
||||
;;
|
||||
|
||||
15
.github/workflows/unittests.yml
vendored
15
.github/workflows/unittests.yml
vendored
@@ -55,6 +55,9 @@ jobs:
|
||||
SFTP_HOST: localhost:2222:/home/testUser1/upload/
|
||||
SFTP_USER: testUser1
|
||||
SFTP_PASS: password
|
||||
CIFS_ADDR: localhost:4445/Data
|
||||
CIFS_USER: samba
|
||||
CIFS_PASSWORD: secret
|
||||
WEBDAV_TEST_BUCKET: 127.0.0.1:9007
|
||||
TIKV_ADDR: 127.0.0.1
|
||||
REDIS_ADDR: redis://127.0.0.1:6379/13
|
||||
@@ -99,12 +102,12 @@ jobs:
|
||||
name: Install redis-cluster
|
||||
uses: vishnudxb/redis-cluster@1.0.5
|
||||
with:
|
||||
master1-port: 7000
|
||||
master2-port: 7001
|
||||
master3-port: 7002
|
||||
slave1-port: 7003
|
||||
slave2-port: 7004
|
||||
slave3-port: 7005
|
||||
master1-port: 7000
|
||||
master2-port: 7001
|
||||
master3-port: 7002
|
||||
slave1-port: 7003
|
||||
slave2-port: 7004
|
||||
slave3-port: 7005
|
||||
|
||||
- name: Prepare Database
|
||||
run: |
|
||||
|
||||
2
Makefile
2
Makefile
@@ -28,7 +28,7 @@ juicefs.cover: Makefile cmd/*.go pkg/*/*.go go.*
|
||||
go build -ldflags="$(LDFLAGS)" -cover -o juicefs .
|
||||
|
||||
juicefs.lite: Makefile cmd/*.go pkg/*/*.go
|
||||
go build -tags nogateway,nowebdav,nocos,nobos,nohdfs,noibmcos,noobs,nooss,noqingstor,nosftp,noswift,noazure,nogs,noufile,nob2,nonfs,nodragonfly,nosqlite,nomysql,nopg,notikv,nobadger,noetcd \
|
||||
go build -tags nogateway,nowebdav,nocos,nobos,nohdfs,noibmcos,noobs,nooss,noqingstor,nosftp,noswift,noazure,nogs,noufile,nob2,nonfs,nodragonfly,nosqlite,nomysql,nopg,notikv,nobadger,noetcd,nocifs \
|
||||
-ldflags="$(LDFLAGS)" -o juicefs.lite .
|
||||
|
||||
juicefs.ceph: Makefile cmd/*.go pkg/*/*.go
|
||||
|
||||
5
go.mod
5
go.mod
@@ -21,6 +21,7 @@ require (
|
||||
github.com/baidubce/bce-sdk-go v0.9.221
|
||||
github.com/bytedance/mockey v1.2.14
|
||||
github.com/ceph/go-ceph v0.18.0
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0
|
||||
github.com/davies/groupcache v0.0.0-20230821031435-e4e8362f58e1
|
||||
github.com/dgraph-io/badger/v4 v4.5.1
|
||||
@@ -147,6 +148,7 @@ require (
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cheggaaa/pb v1.0.29 // indirect
|
||||
github.com/clbanning/mxj v1.8.4 // indirect
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect
|
||||
github.com/coredns/coredns v1.4.0 // indirect
|
||||
github.com/coreos/etcd v3.3.27+incompatible // indirect
|
||||
@@ -172,6 +174,7 @@ require (
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
|
||||
github.com/gammazero/toposort v0.1.1 // indirect
|
||||
github.com/geoffgarside/ber v1.1.0 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.1 // indirect
|
||||
github.com/go-ldap/ldap/v3 v3.2.4 // indirect
|
||||
github.com/go-logr/logr v1.4.2 // indirect
|
||||
@@ -345,3 +348,5 @@ replace github.com/mattn/go-colorable v0.1.6 => github.com/juicedata/go-colorabl
|
||||
replace github.com/mattn/go-colorable v0.1.9 => github.com/juicedata/go-colorable v0.0.0-20250208072043-a97a0c2023db
|
||||
|
||||
replace github.com/mattn/go-colorable v0.0.9 => github.com/juicedata/go-colorable v0.0.0-20250208072043-a97a0c2023db
|
||||
|
||||
replace github.com/cloudsoda/go-smb2 => github.com/juicedata/go-smb2 v0.0.0-20250917090526-d2d0abfb0e05
|
||||
|
||||
6
go.sum
6
go.sum
@@ -170,6 +170,8 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp
|
||||
github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I=
|
||||
github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc h1:0xCWmFKBmarCqqqLeM7jFBSw/Or81UEElFqO8MY+GDs=
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc/go.mod h1:uvR42Hb/t52HQd7x5/ZLzZEK8oihrFpgnodIJ1vte2E=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI=
|
||||
github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
|
||||
@@ -248,6 +250,8 @@ github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uq
|
||||
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
|
||||
github.com/gammazero/toposort v0.1.1 h1:OivGxsWxF3U3+U80VoLJ+f50HcPU1MIqE1JlKzoJ2Eg=
|
||||
github.com/gammazero/toposort v0.1.1/go.mod h1:H2cozTnNpMw0hg2VHAYsAxmkHXBYroNangj2NTBQDvw=
|
||||
github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w=
|
||||
github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.1 h1:pDbRAunXzIUXfx4CB2QJFv5IuPiuoW+sWvr/Us009o8=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno=
|
||||
@@ -476,6 +480,8 @@ github.com/juicedata/go-fuse/v2 v2.1.1-0.20250807045235-112198daa7df h1:H3/AM/YZ
|
||||
github.com/juicedata/go-fuse/v2 v2.1.1-0.20250807045235-112198daa7df/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs=
|
||||
github.com/juicedata/go-nfs-client v0.0.0-20250220101412-d3a8c1ca64a1 h1:GgH2ZG9inMYSme7zZb79z3QeOW70YusbJIVYjvqd508=
|
||||
github.com/juicedata/go-nfs-client v0.0.0-20250220101412-d3a8c1ca64a1/go.mod h1:xOMqi3lOrcGe9uZLnSzgaq94Vc3oz6VPCNDLJUnXpKs=
|
||||
github.com/juicedata/go-smb2 v0.0.0-20250917090526-d2d0abfb0e05 h1:TE+PhAgpUO/mqzHg5Ttq67t3HTraSpR/Es39LjDMJMs=
|
||||
github.com/juicedata/go-smb2 v0.0.0-20250917090526-d2d0abfb0e05/go.mod h1:CgWpFCFWzzEA5hVkhAc6DZZzGd3czx+BblvOzjmg6KA=
|
||||
github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d h1:kpQMvNZJKGY3PTt7OSoahYc4nM0HY67SvK0YyS0GLwA=
|
||||
github.com/juicedata/godaemon v0.0.0-20210629045518-3da5144a127d/go.mod h1:dlxKkLh3qAIPtgr2U/RVzsZJDuXA1ffg+Njikfmhvgw=
|
||||
github.com/juicedata/gogfapi v0.0.0-20241204082332-ecd102647f80 h1:EPg/f3lhbAOjE2M0WpVi47Fk62mEmmPejRuGVdOFQww=
|
||||
|
||||
478
pkg/object/cifs.go
Normal file
478
pkg/object/cifs.go
Normal file
@@ -0,0 +1,478 @@
|
||||
//go:build !nocifs
|
||||
// +build !nocifs
|
||||
|
||||
/*
|
||||
* JuiceFS, Copyright 2025 Juicedata, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package object
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/cloudsoda/go-smb2"
|
||||
)
|
||||
|
||||
type cifsConn struct {
|
||||
session *smb2.Session
|
||||
share *smb2.Share
|
||||
lastUsed time.Time
|
||||
}
|
||||
|
||||
var _ ObjectStorage = (*cifsStore)(nil)
|
||||
var _ FileSystem = (*cifsStore)(nil)
|
||||
|
||||
type cifsStore struct {
|
||||
DefaultObjectStorage
|
||||
host string
|
||||
port string
|
||||
share string
|
||||
user string
|
||||
password string
|
||||
pool chan *cifsConn
|
||||
connIdleTimeout time.Duration
|
||||
}
|
||||
|
||||
// Chmod changes the mode of the file to mode.
|
||||
//
|
||||
// Note: SAMBA protocol has limited support for Unix file permissions.
|
||||
// it controls the FILE_ATTRIBUTE_READONLY attribute. All other permission bits are ignored.
|
||||
//
|
||||
// Examples:
|
||||
// - chmod(0644), chmod(0666), chmod(0755) -> file becomes writable(666)
|
||||
// - chmod(0444), chmod(0400), chmod(0555) -> file becomes read-only(444)
|
||||
//
|
||||
// The returned mode from Stat() will always be either 0666 (writable) or 0444 (read-only)
|
||||
// regardless of the specific mode bits passed to this function.
|
||||
func (c *cifsStore) Chmod(path string, mode os.FileMode) error {
|
||||
return c.withConn(context.Background(), func(conn *cifsConn) error {
|
||||
return conn.share.Chmod(path, mode)
|
||||
})
|
||||
}
|
||||
|
||||
// Chown implements FileSystem.
|
||||
func (c *cifsStore) Chown(path string, owner string, group string) error {
|
||||
return notSupported
|
||||
}
|
||||
|
||||
// Chtimes implements MtimeChanger.
|
||||
func (c *cifsStore) Chtimes(path string, mtime time.Time) error {
|
||||
return c.withConn(context.Background(), func(conn *cifsConn) error {
|
||||
return conn.share.Chtimes(path, time.Time{}, mtime)
|
||||
})
|
||||
}
|
||||
|
||||
func (c *cifsStore) String() string {
|
||||
return fmt.Sprintf("cifs://%s@%s:%s/%s/", c.user, c.host, c.port, c.share)
|
||||
}
|
||||
|
||||
// getConnection returns a CIFS connection from the pool or creates a new one
|
||||
func (c *cifsStore) getConnection(ctx context.Context) (*cifsConn, error) {
|
||||
now := time.Now()
|
||||
for {
|
||||
select {
|
||||
case conn := <-c.pool:
|
||||
if conn.session == nil {
|
||||
continue
|
||||
}
|
||||
// TODO: do it in a new goroutine?
|
||||
if now.Sub(conn.lastUsed) > c.connIdleTimeout {
|
||||
_ = conn.session.Logoff()
|
||||
continue
|
||||
}
|
||||
conn.lastUsed = now
|
||||
return conn, nil
|
||||
default:
|
||||
goto CREATE
|
||||
}
|
||||
}
|
||||
|
||||
CREATE:
|
||||
// Create new connection
|
||||
// FIXME: may create a large number of connection in a short period, exceeding the limit.
|
||||
conn := &cifsConn{}
|
||||
conn.lastUsed = now
|
||||
|
||||
// Establish SMB connection
|
||||
address := net.JoinHostPort(c.host, c.port)
|
||||
d := &smb2.Dialer{
|
||||
Initiator: &smb2.NTLMInitiator{
|
||||
User: c.user,
|
||||
Password: c.password,
|
||||
},
|
||||
}
|
||||
|
||||
var err error
|
||||
conn.session, err = d.Dial(ctx, address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("SMB authentication failed: %v", err)
|
||||
}
|
||||
|
||||
conn.share, err = conn.session.Mount(c.share)
|
||||
if err != nil {
|
||||
_ = conn.session.Logoff()
|
||||
return nil, fmt.Errorf("failed to mount SMB share %s: %v", c.share, err)
|
||||
}
|
||||
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
// releaseConnection returns a connection to the pool or closes it if there's an error
|
||||
func (c *cifsStore) releaseConnection(conn *cifsConn, err error) {
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
select {
|
||||
case c.pool <- conn:
|
||||
return
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// close connection if there's an error or if the pool is full
|
||||
if conn.session != nil {
|
||||
_ = conn.session.Logoff()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *cifsStore) withConn(ctx context.Context, f func(*cifsConn) error) error {
|
||||
conn, err := c.getConnection(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = f(conn)
|
||||
c.releaseConnection(conn, err)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *cifsStore) Head(ctx context.Context, key string) (oj Object, err error) {
|
||||
err = c.withConn(ctx, func(conn *cifsConn) error {
|
||||
fi, err := conn.share.Lstat(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
isSymlink := fi.Mode()&os.ModeSymlink != 0
|
||||
if isSymlink {
|
||||
// SMB doesn't fully support symlinks like POSIX, but we'll try our best
|
||||
fi, err = conn.share.Stat(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
oj = c.fileInfo(key, fi, isSymlink)
|
||||
return nil
|
||||
})
|
||||
return oj, err
|
||||
}
|
||||
|
||||
func (c *cifsStore) Get(ctx context.Context, key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {
|
||||
var readCloser io.ReadCloser
|
||||
err := c.withConn(ctx, func(conn *cifsConn) error {
|
||||
f, err := conn.share.Open(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
finfo, err := f.Stat()
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
if finfo.IsDir() || off > finfo.Size() {
|
||||
_ = f.Close()
|
||||
readCloser = io.NopCloser(bytes.NewBuffer([]byte{}))
|
||||
return nil
|
||||
}
|
||||
if limit > 0 {
|
||||
readCloser = &SectionReaderCloser{
|
||||
SectionReader: io.NewSectionReader(f, off, limit),
|
||||
Closer: f,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
readCloser = f
|
||||
return nil
|
||||
})
|
||||
return readCloser, err
|
||||
}
|
||||
|
||||
func (c *cifsStore) Put(ctx context.Context, key string, in io.Reader, getters ...AttrGetter) (err error) {
|
||||
return c.withConn(ctx, func(conn *cifsConn) error {
|
||||
p := key
|
||||
if strings.HasSuffix(key, dirSuffix) {
|
||||
// perm will not take effect, is not used
|
||||
// ref: https://github.com/cloudsoda/go-smb2/blob/c8e61c7a5fa7bcd1143359f071f9425a9f4dda3f/client.go#L341-L370
|
||||
return conn.share.MkdirAll(p, 0755)
|
||||
}
|
||||
|
||||
var tmp string
|
||||
if PutInplace {
|
||||
tmp = p
|
||||
} else {
|
||||
name := path.Base(p)
|
||||
if len(name) > 200 {
|
||||
name = name[:200]
|
||||
}
|
||||
tmp = path.Join(path.Dir(p), fmt.Sprintf(".%s.tmp.%d", name, rand.Int()))
|
||||
defer func() {
|
||||
if err != nil {
|
||||
_ = conn.share.Remove(tmp)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
f, err := conn.share.Create(tmp)
|
||||
if err != nil && os.IsNotExist(err) {
|
||||
dirPath := path.Dir(p)
|
||||
if dirPath != "/" {
|
||||
err = conn.share.MkdirAll(dirPath, 0755)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
f, err = conn.share.Create(tmp)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
buf := bufPool.Get().(*[]byte)
|
||||
defer bufPool.Put(buf)
|
||||
_, err = io.CopyBuffer(f, in, *buf)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !PutInplace {
|
||||
err = conn.share.Rename(tmp, p)
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func (c *cifsStore) Delete(ctx context.Context, key string, getters ...AttrGetter) (err error) {
|
||||
return c.withConn(ctx, func(conn *cifsConn) error {
|
||||
p := strings.TrimRight(key, dirSuffix)
|
||||
err = conn.share.Remove(p)
|
||||
if err != nil && os.IsNotExist(err) {
|
||||
err = nil
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func (c *cifsStore) fileInfo(key string, fi os.FileInfo, isSymlink bool) Object {
|
||||
owner, group := "nobody", "nobody"
|
||||
ff := &file{
|
||||
obj{key, fi.Size(), fi.ModTime(), fi.IsDir(), ""},
|
||||
owner,
|
||||
group,
|
||||
fi.Mode(),
|
||||
isSymlink,
|
||||
}
|
||||
if fi.IsDir() {
|
||||
if key != "" && !strings.HasSuffix(key, "/") {
|
||||
ff.key += "/"
|
||||
}
|
||||
}
|
||||
return ff
|
||||
}
|
||||
|
||||
func (c *cifsStore) List(ctx context.Context, prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
|
||||
if delimiter != "/" {
|
||||
return nil, false, "", notSupported
|
||||
}
|
||||
|
||||
dir := prefix
|
||||
var objs []Object
|
||||
if !strings.HasSuffix(dir, "/") {
|
||||
dir = path.Dir(dir)
|
||||
if !strings.HasSuffix(dir, dirSuffix) {
|
||||
dir += dirSuffix
|
||||
}
|
||||
} else if marker == "" {
|
||||
obj, err := c.Head(ctx, prefix)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, false, "", nil
|
||||
}
|
||||
return nil, false, "", err
|
||||
}
|
||||
objs = append(objs, obj)
|
||||
}
|
||||
var mEntries []*mEntry
|
||||
err := c.withConn(ctx, func(conn *cifsConn) error {
|
||||
// Ensure directory exists before listing
|
||||
_, err := conn.share.Stat(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Read directory entries
|
||||
entries, err := conn.share.ReadDir(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Process entries
|
||||
mEntries = make([]*mEntry, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
isSymlink := e.Mode()&os.ModeSymlink != 0
|
||||
if e.IsDir() {
|
||||
mEntries = append(mEntries, &mEntry{e, e.Name() + dirSuffix, nil, false})
|
||||
} else if isSymlink && followLink {
|
||||
// SMB doesn't fully support symlinks like POSIX, but we'll try our best
|
||||
fi, err := conn.share.Stat(path.Join(dir, e.Name()))
|
||||
if err != nil {
|
||||
mEntries = append(mEntries, &mEntry{e, e.Name(), nil, true})
|
||||
continue
|
||||
}
|
||||
name := e.Name()
|
||||
if fi.IsDir() {
|
||||
name = e.Name() + dirSuffix
|
||||
}
|
||||
mEntries = append(mEntries, &mEntry{e, name, fi, false})
|
||||
} else {
|
||||
mEntries = append(mEntries, &mEntry{e, e.Name(), nil, isSymlink})
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if os.IsNotExist(err) || os.IsPermission(err) {
|
||||
logger.Warnf("skip %s: %s", dir, err)
|
||||
return nil, false, "", nil
|
||||
}
|
||||
|
||||
// Sort entries by name
|
||||
sort.Slice(mEntries, func(i, j int) bool { return mEntries[i].Name() < mEntries[j].Name() })
|
||||
|
||||
// Generate object list
|
||||
for _, e := range mEntries {
|
||||
p := path.Join(dir, e.Name())
|
||||
if e.IsDir() && !strings.HasSuffix(p, "/") {
|
||||
p = p + "/"
|
||||
}
|
||||
key := p
|
||||
if !strings.HasPrefix(key, prefix) || (marker != "" && key <= marker) {
|
||||
continue
|
||||
}
|
||||
|
||||
info := e.Info()
|
||||
f := c.fileInfo(key, info, e.isSymlink)
|
||||
objs = append(objs, f)
|
||||
if len(objs) == int(limit) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return generateListResult(objs, limit)
|
||||
}
|
||||
|
||||
func (c *cifsStore) Copy(ctx context.Context, dst, src string) error {
|
||||
r, err := c.Get(ctx, src, 0, -1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer r.Close()
|
||||
return c.Put(ctx, dst, r)
|
||||
}
|
||||
|
||||
func parseEndpoint(endpoint string) (host, port, share string, err error) {
|
||||
if !strings.Contains(endpoint, "://") {
|
||||
endpoint = "cifs://" + endpoint
|
||||
}
|
||||
u, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if u.Scheme != "" && (u.Scheme != "cifs" && u.Scheme != "smb") {
|
||||
err = fmt.Errorf("invalid scheme %s, should be cifs:// or smb://", u.Scheme)
|
||||
return
|
||||
}
|
||||
|
||||
host = u.Hostname()
|
||||
port = u.Port()
|
||||
if port == "" {
|
||||
port = "445" // Default SMB port
|
||||
}
|
||||
parts := strings.Split(u.Path, "/")
|
||||
if len(parts) < 2 || parts[1] == "" {
|
||||
err = fmt.Errorf("endpoint should be a valid share name (%s)", "\\\\<server>\\<share>")
|
||||
return
|
||||
}
|
||||
if len(parts) > 2 && parts[2] != "" {
|
||||
err = fmt.Errorf("endpoint should be a valid share name (%s)", "\\\\<server>\\<share>")
|
||||
return
|
||||
}
|
||||
share = parts[1]
|
||||
return
|
||||
}
|
||||
|
||||
func newCifs(endpoint, username, password, _ string) (ObjectStorage, error) {
|
||||
host, port, share, err := parseEndpoint(endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if username == "" {
|
||||
return nil, fmt.Errorf("CIFS username/ak is required")
|
||||
}
|
||||
|
||||
if password == "" {
|
||||
return nil, fmt.Errorf("CIFS password/sk is required")
|
||||
}
|
||||
|
||||
store := &cifsStore{
|
||||
host: host,
|
||||
port: port,
|
||||
share: share,
|
||||
user: username,
|
||||
password: password,
|
||||
connIdleTimeout: 5 * time.Minute,
|
||||
pool: make(chan *cifsConn, 8),
|
||||
}
|
||||
|
||||
// Test connection
|
||||
conn, err := store.getConnection(context.Background())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
store.releaseConnection(conn, nil)
|
||||
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
// Allow both cifs:// and smb:// schemes
|
||||
Register("cifs", newCifs)
|
||||
Register("smb", newCifs)
|
||||
}
|
||||
@@ -64,6 +64,18 @@ func TestSftp2(t *testing.T) { //skip mutate
|
||||
testFileSystem(t, sftp)
|
||||
}
|
||||
|
||||
func TestCifs2(t *testing.T) { //skip mutate
|
||||
if os.Getenv("CIFS_ADDR") == "" {
|
||||
fmt.Println("skip CIFS test")
|
||||
t.SkipNow()
|
||||
}
|
||||
cifs, err := newCifs(os.Getenv("CIFS_ADDR"), os.Getenv("CIFS_USER"), os.Getenv("CIFS_PASSWORD"), "")
|
||||
if err != nil {
|
||||
t.Fatalf("create: %s", err)
|
||||
}
|
||||
testFileSystem(t, cifs)
|
||||
}
|
||||
|
||||
func TestHDFS2(t *testing.T) { //skip mutate
|
||||
if os.Getenv("HDFS_ADDR") == "" {
|
||||
t.Skip()
|
||||
@@ -157,6 +169,9 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
|
||||
if _, ok := ss.(*nfsStore); ok {
|
||||
expectedKeys = []string{"x/", "x/x.txt", "xy.txt", "xyz/", "xyz/xyz.txt"}
|
||||
}
|
||||
if _, ok := ss.(*cifsStore); ok {
|
||||
expectedKeys = []string{"x/", "x/x.txt", "xy.txt", "xyz/", "xyz/xyz.txt"}
|
||||
}
|
||||
if mode == 0422 {
|
||||
if strings.HasPrefix(s.String(), "gluster://") {
|
||||
expectedKeys = []string{"x/", "x/x.txt", "xy.txt", "xyz/", "xyz/xyz.txt"}
|
||||
|
||||
@@ -1093,6 +1093,18 @@ func TestDragonfly(t *testing.T) { //skip mutate
|
||||
testStorage(t, dragonfly)
|
||||
}
|
||||
|
||||
func TestCifs(t *testing.T) { //skip mutate
|
||||
if os.Getenv("CIFS_ADDR") == "" {
|
||||
fmt.Println("skip CIFS test")
|
||||
t.SkipNow()
|
||||
}
|
||||
cifs, err := newCifs(os.Getenv("CIFS_ADDR"), os.Getenv("CIFS_USER"), os.Getenv("CIFS_PASSWORD"), "")
|
||||
if err != nil {
|
||||
t.Fatalf("create: %s", err)
|
||||
}
|
||||
testStorage(t, cifs)
|
||||
}
|
||||
|
||||
// func TestBunny(t *testing.T) { //skip mutate
|
||||
// if os.Getenv("BUNNY_ENDPOINT") == "" {
|
||||
// t.SkipNow()
|
||||
|
||||
Reference in New Issue
Block a user