Skip to content

Commit 95a5fd4

Browse files
dzbarskydzbarsky
committed
try to share code
1 parent fdf804f commit 95a5fd4

File tree

7 files changed

+147
-226
lines changed

7 files changed

+147
-226
lines changed

‎tools/common/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ go_library(
77
"clonefile_stub.go",
88
"copy.go",
99
"file.go",
10+
"walker.go",
1011
],
1112
importpath = "github.com/bazel-contrib/bazel-lib/tools/common",
1213
visibility = ["//visibility:public"],

‎tools/common/copy.go

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -27,22 +27,22 @@ func CopyFile(src string, dst string) error {
2727
return err
2828
}
2929

30-
func Copy(opts CopyOpts) {
30+
func Copy(opts CopyFileOpts) {
3131
if !opts.srcInfo.Mode().IsRegular() {
3232
log.Fatalf("%s is not a regular file", opts.src)
3333
}
3434

3535
opModifier := ""
3636
const fallback = " (fallback)"
3737

38-
if opts.hardlink {
38+
if opts.Hardlink {
3939
// hardlink this file
40-
if opts.verbose {
40+
if opts.Verbose {
4141
fmt.Printf("hardlink%s %v => %v\n", opModifier, opts.src, opts.dst)
4242
}
4343
err := os.Link(opts.src, opts.dst)
4444
if err != nil {
45-
if opts.verbose {
45+
if opts.Verbose {
4646
fmt.Printf("hardlink failed: %v\n", err)
4747
opModifier = fallback
4848
}
@@ -53,35 +53,35 @@ func Copy(opts CopyOpts) {
5353
}
5454

5555
// clone this file
56-
if opts.verbose {
56+
if opts.Verbose {
5757
fmt.Printf("clonefile%s %v => %v\n", opModifier, opts.src, opts.dst)
5858
}
5959
switch supported, err := cloneFile(opts.src, opts.dst); {
6060
case !supported:
61-
if opts.verbose {
61+
if opts.Verbose {
6262
fmt.Print("clonefile skipped: not supported by platform\n")
6363
}
6464
// fallback to copy
6565
case supported && err == nil:
6666
return
6767
case supported && err != nil:
68-
if opts.verbose {
68+
if opts.Verbose {
6969
fmt.Printf("clonefile failed: %v\n", err)
7070
opModifier = fallback
7171
}
7272
// fallback to copy
7373
}
7474

7575
// copy this file
76-
if opts.verbose {
76+
if opts.Verbose {
7777
fmt.Printf("copy%s %v => %v\n", opModifier, opts.src, opts.dst)
7878
}
7979
err := CopyFile(opts.src, opts.dst)
8080
if err != nil {
8181
log.Fatal(err)
8282
}
8383

84-
if opts.preserveMTime {
84+
if opts.PreserveMTime {
8585
accessTime := time.Now()
8686
err := os.Chtimes(opts.dst, accessTime, opts.srcInfo.ModTime())
8787
if err != nil {
@@ -91,10 +91,10 @@ func Copy(opts CopyOpts) {
9191
}
9292

9393
type CopyWorker struct {
94-
queue <-chan CopyOpts
94+
queue <-chan CopyFileOpts
9595
}
9696

97-
func NewCopyWorker(queue <-chan CopyOpts) *CopyWorker {
97+
func NewCopyWorker(queue <-chan CopyFileOpts) *CopyWorker {
9898
return &CopyWorker{queue: queue}
9999
}
100100

@@ -106,13 +106,17 @@ func (w *CopyWorker) Run(wg *sync.WaitGroup) {
106106
}
107107

108108
type CopyOpts struct {
109-
src, dst string
110-
srcInfo fs.FileInfo
111-
hardlink bool
112-
verbose bool
113-
preserveMTime bool
109+
Hardlink bool
110+
Verbose bool
111+
PreserveMTime bool
114112
}
115113

116-
func NewCopyOpts(src string, dst string, srcInfo fs.FileInfo, hardlink bool, verbose bool, preserveMTime bool) CopyOpts {
117-
return CopyOpts{src: src, dst: dst, srcInfo: srcInfo, hardlink: hardlink, verbose: verbose, preserveMTime: preserveMTime}
114+
type CopyFileOpts struct {
115+
CopyOpts
116+
src, dst string
117+
srcInfo fs.FileInfo
118+
}
119+
120+
func NewCopyFileOpts(src string, dst string, srcInfo fs.FileInfo, copyOpts CopyOpts) CopyFileOpts {
121+
return CopyFileOpts{src: src, dst: dst, srcInfo: srcInfo, CopyOpts: copyOpts}
118122
}

‎tools/common/walker.go

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
package common
2+
3+
import (
4+
"fmt"
5+
"io/fs"
6+
"os"
7+
"path/filepath"
8+
"sync"
9+
)
10+
11+
type Walker struct {
12+
queue chan<- CopyFileOpts
13+
wg sync.WaitGroup
14+
}
15+
16+
func NewWalker(queueDepth int, numWorkers int) *Walker {
17+
queue := make(chan CopyFileOpts, 100)
18+
var wg sync.WaitGroup
19+
20+
wg.Add(numWorkers)
21+
for i := 0; i < numWorkers; i++ {
22+
go NewCopyWorker(queue).Run(&wg)
23+
}
24+
25+
return &Walker{queue, wg}
26+
}
27+
28+
func (w *Walker) Close() {
29+
close(w.queue)
30+
w.wg.Wait()
31+
}
32+
33+
func (w *Walker) CopyFile(src string, dst string, info fs.FileInfo, opts CopyOpts) {
34+
w.queue <- NewCopyFileOpts(src, dst, info, opts)
35+
}
36+
37+
func (w *Walker) CopyDir(src string, dst string, opts CopyOpts) error {
38+
srcPaths := map[string]bool{}
39+
// filepath.WalkDir walks the file tree rooted at root, calling fn for each file or directory in
40+
// the tree, including root. See https://pkg.go.dev/path/filepath#WalkDir for more info.
41+
return filepath.WalkDir(src, func(p string, dirEntry fs.DirEntry, err error) error {
42+
if err != nil {
43+
return err
44+
}
45+
46+
r, err := FileRel(src, p)
47+
if err != nil {
48+
return err
49+
}
50+
51+
d := filepath.Join(dst, r)
52+
53+
if dirEntry.IsDir() {
54+
srcPaths[src] = true
55+
return os.MkdirAll(d, os.ModePerm)
56+
}
57+
58+
info, err := dirEntry.Info()
59+
if err != nil {
60+
return err
61+
}
62+
63+
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
64+
// symlink to directories are intentionally never followed by filepath.Walk to avoid infinite recursion
65+
linkPath, err := Realpath(p)
66+
if err != nil {
67+
if os.IsNotExist(err) {
68+
return fmt.Errorf("failed to get realpath of dangling symlink %s: %w", p, err)
69+
}
70+
return fmt.Errorf("failed to get realpath of %s: %w", p, err)
71+
}
72+
if srcPaths[linkPath] {
73+
// recursive symlink; silently ignore
74+
return nil
75+
}
76+
stat, err := os.Stat(linkPath)
77+
if err != nil {
78+
return fmt.Errorf("failed to stat file %s pointed to by symlink %s: %w", linkPath, p, err)
79+
}
80+
if stat.IsDir() {
81+
// symlink points to a directory
82+
return w.CopyDir(linkPath, d, opts)
83+
} else {
84+
// symlink points to a regular file
85+
w.queue <- NewCopyFileOpts(linkPath, d, stat, opts)
86+
return nil
87+
}
88+
}
89+
90+
// a regular file
91+
w.queue <- NewCopyFileOpts(p, d, info, opts)
92+
return nil
93+
})
94+
}

‎tools/copy_directory/main.go

Lines changed: 11 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -2,86 +2,12 @@ package main
22

33
import (
44
"fmt"
5-
"io/fs"
65
"log"
76
"os"
8-
"path/filepath"
9-
"sync"
107

118
"github.com/bazel-contrib/bazel-lib/tools/common"
129
)
1310

14-
type pathSet map[string]bool
15-
16-
var (
17-
srcPaths = pathSet{}
18-
hardlink = false
19-
verbose = false
20-
preserveMTime = false
21-
)
22-
23-
type walker struct {
24-
queue chan<- common.CopyOpts
25-
}
26-
27-
func (w *walker) copyDir(src string, dst string) error {
28-
// filepath.WalkDir walks the file tree rooted at root, calling fn for each file or directory in
29-
// the tree, including root. See https://pkg.go.dev/path/filepath#WalkDir for more info.
30-
return filepath.WalkDir(src, func(p string, dirEntry fs.DirEntry, err error) error {
31-
if err != nil {
32-
return err
33-
}
34-
35-
r, err := common.FileRel(src, p)
36-
if err != nil {
37-
return err
38-
}
39-
40-
d := filepath.Join(dst, r)
41-
42-
if dirEntry.IsDir() {
43-
srcPaths[src] = true
44-
return os.MkdirAll(d, os.ModePerm)
45-
}
46-
47-
info, err := dirEntry.Info()
48-
if err != nil {
49-
return err
50-
}
51-
52-
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
53-
// symlink to directories are intentionally never followed by filepath.Walk to avoid infinite recursion
54-
linkPath, err := common.Realpath(p)
55-
if err != nil {
56-
if os.IsNotExist(err) {
57-
return fmt.Errorf("failed to get realpath of dangling symlink %s: %w", p, err)
58-
}
59-
return fmt.Errorf("failed to get realpath of %s: %w", p, err)
60-
}
61-
if srcPaths[linkPath] {
62-
// recursive symlink; silently ignore
63-
return nil
64-
}
65-
stat, err := os.Stat(linkPath)
66-
if err != nil {
67-
return fmt.Errorf("failed to stat file %s pointed to by symlink %s: %w", linkPath, p, err)
68-
}
69-
if stat.IsDir() {
70-
// symlink points to a directory
71-
return w.copyDir(linkPath, d)
72-
} else {
73-
// symlink points to a regular file
74-
w.queue <- common.NewCopyOpts(linkPath, d, stat, hardlink, verbose, preserveMTime)
75-
return nil
76-
}
77-
}
78-
79-
// a regular file
80-
w.queue <- common.NewCopyOpts(p, d, info, hardlink, verbose, preserveMTime)
81-
return nil
82-
})
83-
}
84-
8511
func main() {
8612
args := os.Args[1:]
8713

@@ -93,31 +19,23 @@ func main() {
9319
src := args[0]
9420
dst := args[1]
9521

22+
var opts common.CopyOpts
9623
if len(args) > 2 {
9724
for _, a := range os.Args[2:] {
98-
if a == "--hardlink" {
99-
hardlink = true
100-
} else if a == "--verbose" {
101-
verbose = true
102-
} else if a == "--preserve-mtime" {
103-
preserveMTime = true
25+
switch a {
26+
case "--hardlink":
27+
opts.Hardlink = true
28+
case "--verbose":
29+
opts.Verbose = true
30+
case "--preserve-mtime":
31+
opts.PreserveMTime = true
10432
}
10533
}
10634
}
10735

108-
queue := make(chan common.CopyOpts, 100)
109-
var wg sync.WaitGroup
110-
111-
const numWorkers = 10
112-
wg.Add(numWorkers)
113-
for i := 0; i < numWorkers; i++ {
114-
go common.NewCopyWorker(queue).Run(&wg)
115-
}
116-
117-
walker := &walker{queue}
118-
if err := walker.copyDir(src, dst); err != nil {
36+
walker := common.NewWalker(100, 10)
37+
if err := walker.CopyDir(src, dst, opts); err != nil {
11938
log.Fatal(err)
12039
}
121-
close(queue)
122-
wg.Wait()
40+
walker.Close()
12341
}

‎tools/copy_to_directory/main.go

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ func longestGlobMatch(g string, test string) (string, error) {
124124
}
125125

126126
type walker struct {
127-
queue chan<- common.CopyOpts
127+
queue chan<- common.CopyFileOpts
128128
}
129129

130130
func (w *walker) copyDir(cfg *config, srcPaths pathSet, file fileInfo) error {
@@ -322,7 +322,9 @@ func (w *walker) copyPath(cfg *config, file fileInfo, outputPath string) error {
322322

323323
if !cfg.AllowOverwrites {
324324
// if we don't allow overwrites then we can start copying as soon as a copy is calculated
325-
w.queue <- common.NewCopyOpts(file.Path, outputPath, file.FileInfo, file.Hardlink, cfg.Verbose, cfg.PreserveMTime)
325+
w.queue <- common.NewCopyFileOpts(file.Path, outputPath, file.FileInfo,
326+
common.CopyOpts{Hardlink: file.Hardlink, Verbose: cfg.Verbose, PreserveMTime: cfg.PreserveMTime},
327+
)
326328
}
327329

328330
return nil
@@ -406,7 +408,7 @@ func main() {
406408
log.Fatal(err)
407409
}
408410

409-
queue := make(chan common.CopyOpts, 100)
411+
queue := make(chan common.CopyFileOpts, 100)
410412
var wg sync.WaitGroup
411413

412414
const numWorkers = 10
@@ -424,7 +426,9 @@ func main() {
424426
// if we allow overwrites then we must wait until all copy paths are calculated before starting
425427
// any copy operations
426428
for outputPath, file := range copySet {
427-
queue <- common.NewCopyOpts(file.Path, outputPath, file.FileInfo, file.Hardlink, cfg.Verbose, cfg.PreserveMTime)
429+
queue <- common.NewCopyFileOpts(file.Path, outputPath, file.FileInfo,
430+
common.CopyOpts{Hardlink: file.Hardlink, Verbose: cfg.Verbose, PreserveMTime: cfg.PreserveMTime},
431+
)
428432
}
429433
}
430434

‎tools/write_source_files/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ go_library(
55
srcs = ["main.go"],
66
importpath = "github.com/bazel-contrib/bazel-lib/tools/write_source_files",
77
visibility = ["//visibility:public"],
8+
deps = ["//tools/common"],
89
)
910

1011
go_binary(

0 commit comments

Comments
 (0)