Skip to content

Commit de43c5a

Browse files
fclairambfclairamb
authored
Cleaning up logging around S&D & Gdrive (#412)
* Adding sync & delete support
1 parent bc2187f commit de43c5a

11 files changed

Lines changed: 142 additions & 13 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ ftpserver
33
.idea
44
.vscode
55
*.json
6+
__debug__bin

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ At the current stage, supported backend are:
1919
- [SFTP](https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol) through [afero's sftpfs](https://github.com/spf13/afero/)
2020
- Email through [go-mail](https://github.com/go-mail/mail) thanks to [@x-way](https://github.com/x-way)
2121

22+
And with those are supported common parameters to switch them to read-only, enable logging access, or use a temporary directory file (see [doc](https://github.com/fclairamb/ftpserver/tree/master/fs)).
23+
2224
## Current status of the project
2325

2426
### Features

‎config/config.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
"errors"
77
"os"
88

9-
"github.com/fclairamb/ftpserverlib/log"
9+
log "github.com/fclairamb/go-log"
1010

1111
"github.com/fclairamb/ftpserver/config/confpar"
1212
"github.com/fclairamb/ftpserver/fs"

‎fs/README.md‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
# File system loading
2+
3+
On top of the supported file system and their associated config,
4+
they are few parameters that can be enabled.
5+
6+
## Read-only
7+
This blocks any writing call to the underlying file system. It can be abled by the `read_only` boolean parameter.
8+
9+
```json
10+
{
11+
"version": 1,
12+
"accesses": [
13+
{
14+
"read_only": true,
15+
16+
// The usual FS config:
17+
"user": "test",
18+
"pass": "test",
19+
"fs": "os",
20+
"params": {
21+
"basePath": "/tmp"
22+
}
23+
}
24+
]
25+
}
26+
```
27+
28+
## Shared
29+
This makes sure the underlying filesystem is loaded once per use. It can be enabled by the `shared` boolean parameter.
30+
31+
```json
32+
{
33+
"version": 1,
34+
"accesses": [
35+
{
36+
"shared": true,
37+
// The usual FS config:
38+
"user": "test",
39+
"pass": "test",
40+
"fs": "os",
41+
"params": {
42+
"basePath": "/tmp"
43+
}
44+
}
45+
]
46+
}
47+
```
48+
49+
## Sync & Delete
50+
This creates a temporary local folder that will be replicated to a destination file system.
51+
52+
53+
```json
54+
{
55+
"version": 1,
56+
"accesses": [
57+
{
58+
"sync_and_delete": {
59+
"enable": true, // To enable it
60+
"directory": "/tmp/dir" // Temporary directory (optional)
61+
},
62+
// The usual FS config:
63+
"user": "test",
64+
"pass": "test",
65+
"fs": "os",
66+
"params": {
67+
"basePath": "/target"
68+
}
69+
}
70+
]
71+
}
72+
```
73+
74+
## Logging
75+
```json
76+
{
77+
"version": 1,
78+
"accesses": [
79+
{
80+
"logging": {
81+
"file_accesses": true, // This logs every single file access
82+
"ftp_exchanges": true // This logs every single FTP command
83+
},
84+
// The usual FS config:
85+
"user": "test",
86+
"pass": "test",
87+
"fs": "os",
88+
"params": {
89+
"basePath": "/target"
90+
}
91+
}
92+
]
93+
}
94+
```

‎fs/fs.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55
"fmt"
66

77
snd "github.com/fclairamb/afero-snd"
8-
"github.com/fclairamb/ftpserverlib/log"
8+
log "github.com/fclairamb/go-log"
99
"github.com/spf13/afero"
1010

1111
"github.com/fclairamb/ftpserver/config/confpar"
@@ -42,7 +42,7 @@ func LoadFs(access *confpar.Access, logger log.Logger) (afero.Fs, error) {
4242
case "mail":
4343
fs, err = mail.LoadFs(access)
4444
case "gdrive":
45-
fs, err = gdrive.LoadFs(access, logger)
45+
fs, err = gdrive.LoadFs(access, logger.With("component", "gdrive"))
4646
case "dropbox":
4747
fs, err = dropbox.LoadFs(access)
4848
default:
@@ -64,6 +64,7 @@ func LoadFs(access *confpar.Access, logger log.Logger) (afero.Fs, error) {
6464
fs, err = snd.NewFs(&snd.Config{
6565
Destination: fs,
6666
Temporary: temp,
67+
Logger: logger.With("component", "snd"),
6768
})
6869
}
6970

‎fs/fslog/fslog.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import (
77

88
"github.com/spf13/afero"
99

10-
"github.com/fclairamb/ftpserverlib/log"
10+
log "github.com/fclairamb/go-log"
1111
)
1212

1313
// File is a wrapper to log interactions around file accesses

‎fs/gdrive/gdrive.go‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99

1010
drv "github.com/fclairamb/afero-gdrive"
1111
drvoa "github.com/fclairamb/afero-gdrive/oauthhelper"
12-
"github.com/fclairamb/ftpserverlib/log"
12+
log "github.com/fclairamb/go-log"
1313
"github.com/spf13/afero"
1414
"golang.org/x/oauth2"
1515

@@ -22,7 +22,6 @@ var ErrMissingGoogleClientCredentials = errors.New("missing the google client cr
2222

2323
// LoadFs loads a file system from an access description
2424
func LoadFs(access *confpar.Access, logger log.Logger) (afero.Fs, error) {
25-
logger = logger.With("fs", "gdrive")
2625
googleClientID := access.Params["google_client_id"]
2726
googleClientSecret := access.Params["google_client_secret"]
2827
tokenFile := access.Params["token_file"]
@@ -80,10 +79,15 @@ func LoadFs(access *confpar.Access, logger log.Logger) (afero.Fs, error) {
8079
}
8180

8281
if saveToken {
83-
if err := drvoa.StoreTokenToFile(tokenFile, auth.Token); err != nil {
84-
return nil, fmt.Errorf("token couldn't be saved: %w", err)
82+
if errStoreToken := drvoa.StoreTokenToFile(tokenFile, auth.Token); errStoreToken != nil {
83+
return nil, fmt.Errorf("token couldn't be saved: %w", errStoreToken)
8584
}
8685
}
8786

88-
return drv.New(httpClient)
87+
gdriveFs, err := drv.New(httpClient)
88+
if err == nil {
89+
gdriveFs.Logger = logger
90+
}
91+
92+
return gdriveFs, err
8993
}

‎go.mod‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,39 @@ module github.com/fclairamb/ftpserver
33
go 1.16
44

55
require (
6+
github.com/apache/thrift v0.13.0 // indirect
7+
github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a // indirect
8+
github.com/aws/aws-lambda-go v1.13.3 // indirect
69
github.com/aws/aws-sdk-go v1.40.7
710
github.com/fclairamb/afero-dropbox v0.1.0
811
github.com/fclairamb/afero-gdrive v0.3.0
912
github.com/fclairamb/afero-s3 v0.3.0
10-
github.com/fclairamb/afero-snd v0.0.0-20210725192044-a2d6c522f7ed
11-
github.com/fclairamb/ftpserverlib v0.14.0
13+
github.com/fclairamb/afero-snd v0.1.0
14+
github.com/fclairamb/ftpserverlib v0.14.1-0.20210727203929-61f871ae46fc
15+
github.com/fclairamb/go-log v0.1.0
1216
github.com/go-mail/mail v2.3.1+incompatible
17+
github.com/go-sql-driver/mysql v1.4.0 // indirect
18+
github.com/hashicorp/go-version v1.2.0 // indirect
19+
github.com/hashicorp/go.net v0.0.1 // indirect
20+
github.com/lightstep/lightstep-tracer-go v0.18.1 // indirect
21+
github.com/mitchellh/gox v0.4.0 // indirect
22+
github.com/mitchellh/iochan v1.0.0 // indirect
23+
github.com/oklog/oklog v0.3.2 // indirect
24+
github.com/oklog/run v1.0.0 // indirect
25+
github.com/opentracing/basictracer-go v1.0.0 // indirect
26+
github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5 // indirect
27+
github.com/pact-foundation/pact-go v1.0.4 // indirect
28+
github.com/pborman/uuid v1.2.0 // indirect
1329
github.com/pkg/sftp v1.13.2
30+
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da // indirect
1431
github.com/spf13/afero v1.6.0
32+
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 // indirect
33+
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee // indirect
1534
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97
1635
golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914
1736
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
1837
gopkg.in/mail.v2 v2.3.1 // indirect
38+
sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 // indirect
1939
)
2040

2141
// replace github.com/fclairamb/ftpserverlib => /Users/florent/go/src/github.com/fclairamb/ftpserverlib

‎go.sum‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,10 +131,17 @@ github.com/fclairamb/afero-s3 v0.3.0 h1:yqgJDvVY9XHetHqSrUkXy/YoNWpVvuCRX7yfC6pc
131131
github.com/fclairamb/afero-s3 v0.3.0/go.mod h1:jOjRl8+IamjtYgZTDagbLP6+cdarjuigsm6V7cP4LZA=
132132
github.com/fclairamb/afero-snd v0.0.0-20210725192044-a2d6c522f7ed h1:sCJUKutjhNiGMCB5ElkKScjP9Xh5TiLe9FutIUJMnJY=
133133
github.com/fclairamb/afero-snd v0.0.0-20210725192044-a2d6c522f7ed/go.mod h1:3KupVhrtzquMmydX3IGUbhLVbW/KXrg9QonyJtmHITY=
134+
github.com/fclairamb/afero-snd v0.1.0 h1:eQ28zAnjamm+4wxl/NlzP7LYZaSTx9fNqPHPvNLfQE0=
135+
github.com/fclairamb/afero-snd v0.1.0/go.mod h1:3KupVhrtzquMmydX3IGUbhLVbW/KXrg9QonyJtmHITY=
134136
github.com/fclairamb/ftpserverlib v0.14.0 h1:hF7cOVgihzmUwC4+i31iZ8MeCwK5IUipSZEDi4g6G4w=
135137
github.com/fclairamb/ftpserverlib v0.14.0/go.mod h1:ATLgn4bHgiM9+vfZbK+rMu/dqgkxO5nk94x/9f8ffDI=
138+
github.com/fclairamb/ftpserverlib v0.14.1-0.20210727203929-61f871ae46fc h1:U5mPcdTTchLyqeQxOtizTg74PxSkJqSpVRFyDJRi5mM=
139+
github.com/fclairamb/ftpserverlib v0.14.1-0.20210727203929-61f871ae46fc/go.mod h1:KNoFofEUEz2PvH8vzZi+lUWZ2/NgCIdAjL0s+AMTbRQ=
136140
github.com/fclairamb/go-log v0.0.0-20210717204555-d370617e3635 h1:LnQGTX50HKMdM3yGt5RlYN4JR8gOYuwa+qB4LkSzfuM=
137141
github.com/fclairamb/go-log v0.0.0-20210717204555-d370617e3635/go.mod h1:iqmym8aI6xBbZXnZSPjElrmQrlEwjwEemOmIzKaTBM8=
142+
github.com/fclairamb/go-log v0.0.0-20210725225107-80efc81cb386/go.mod h1:iqmym8aI6xBbZXnZSPjElrmQrlEwjwEemOmIzKaTBM8=
143+
github.com/fclairamb/go-log v0.1.0 h1:fNoqk8w62i4EDEuRzDgHdDVTqMYSyr3DS981R7F2x/Y=
144+
github.com/fclairamb/go-log v0.1.0/go.mod h1:iqmym8aI6xBbZXnZSPjElrmQrlEwjwEemOmIzKaTBM8=
138145
github.com/fclairamb/go-log v0.0.0-20210725225107-80efc81cb386 h1:f/v/Fr+f7LiOKbMJlBuQPRmqrsYUXDzm1dpCJ29j9yo=
139146
github.com/fclairamb/go-log v0.0.0-20210725225107-80efc81cb386/go.mod h1:iqmym8aI6xBbZXnZSPjElrmQrlEwjwEemOmIzKaTBM8=
140147
github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=

‎main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
"time"
1111

1212
ftpserver "github.com/fclairamb/ftpserverlib"
13-
"github.com/fclairamb/ftpserverlib/log/gokit"
13+
"github.com/fclairamb/go-log/gokit"
1414

1515
"github.com/fclairamb/ftpserver/config"
1616
"github.com/fclairamb/ftpserver/server"

0 commit comments

Comments
 (0)