diff --git a/README.md b/README.md index b99bb1c..9ae5d5e 100644 --- a/README.md +++ b/README.md @@ -111,20 +111,22 @@ Flags: --ignore-name=".chkbitignore" filename that chkbit reads its ignore list from, needs to start with '.' (default: .chkbitignore) - --index-db use a index database instead of index files + --db use a index database instead of index files -w, --workers=5 number of workers to use (default: 5) --[no-]plain show plain status instead of being fancy -q, --[no-]quiet quiet, don't show progress/information -v, --[no-]verbose verbose output -V, --version show version information -mode - -c, --check check mode: chkbit will verify files in readonly - mode (default mode) - -u, --update update mode: add and update indices - -a, --add-only add mode: only add new and modified files, do not - check existing (quicker) - -i, --show-ignored-only show-ignored mode: only show ignored files +Mode + -c, --check chkbit will verify files in readonly mode (default + mode) + -u, --update add and update indices + -a, --add-only only add new and modified files, do not check + existing (quicker) + --init-db initialize a new index database at the given path + for use with --db + -i, --show-ignored-only only show ignored files ``` ``` @@ -188,6 +190,14 @@ Add a `.chkbitignore` file containing the names of the files/directories you wis - you can use `path/sub/name` to ignore a file/directory in a sub path - hidden files (starting with a `.`) are ignored by default unless you use the `-d` option +## Database Usage + +*experimental* + +To use a chkbit database you need to initalize it first with `chkbit --init-db PATH`. + +Then you can run `chkbit` on anything below PATH and it will be tracked in the database. + ## chkbit as a Go module diff --git a/cmd/chkbit/main.go b/cmd/chkbit/main.go index 3e4bad0..653493f 100644 --- a/cmd/chkbit/main.go +++ b/cmd/chkbit/main.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "io" "log" @@ -45,10 +46,11 @@ var ( var cli struct { Paths []string `arg:"" optional:"" name:"paths" help:"directories to check"` Tips bool `short:"H" help:"Show tips."` - Check bool `short:"c" help:"check mode: chkbit will verify files in readonly mode (default mode)" xor:"mode" group:"mode"` - Update bool `short:"u" help:"update mode: add and update indices" xor:"mode" group:"mode"` - AddOnly bool `short:"a" help:"add mode: only add new and modified files, do not check existing (quicker)" xor:"mode" group:"mode"` - ShowIgnoredOnly bool `short:"i" help:"show-ignored mode: only show ignored files" xor:"mode" group:"mode"` + Check bool `short:"c" help:"chkbit will verify files in readonly mode (default mode)" xor:"mode" group:"Mode"` + Update bool `short:"u" help:"add and update indices" xor:"mode" group:"Mode"` + AddOnly bool `short:"a" help:"only add new and modified files, do not check existing (quicker)" xor:"mode" group:"Mode"` + InitDb bool `help:"initialize a new index database at the given path for use with --db" xor:"mode" group:"Mode"` + ShowIgnoredOnly bool `short:"i" help:"only show ignored files" xor:"mode" group:"Mode"` ShowMissing bool `short:"m" help:"show missing files/directories" negatable:""` IncludeDot bool `short:"d" help:"include dot files" negatable:""` SkipSymlinks bool `short:"S" help:"do not follow symlinks" negatable:""` @@ -60,7 +62,7 @@ var cli struct { Algo string `default:"blake3" help:"hash algorithm: md5, sha512, blake3 (default: blake3)"` IndexName string `default:".chkbit" help:"filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit)"` IgnoreName string `default:".chkbitignore" help:"filename that chkbit reads its ignore list from, needs to start with '.' (default: .chkbitignore)"` - IndexDb bool `help:"use a index database instead of index files"` + Db bool `help:"use a index database instead of index files"` Workers int `short:"w" default:"5" help:"number of workers to use (default: 5)"` Plain bool `help:"show plain status instead of being fancy" negatable:""` Quiet bool `short:"q" help:"quiet, don't show progress/information" negatable:""` @@ -159,19 +161,17 @@ func (m *Main) showStatus() { } } -func (m *Main) process() bool { +func (m *Main) process() (bool, error) { // verify mode var b01 = map[bool]int8{false: 0, true: 1} if b01[cli.Check]+b01[cli.Update]+b01[cli.AddOnly]+b01[cli.ShowIgnoredOnly] > 1 { - fmt.Println("Error: can only run one mode at a time!") - os.Exit(1) + return false, errors.New("can only run one mode at a time") } var err error m.context, err = chkbit.NewContext(cli.Workers, cli.Algo, cli.IndexName, cli.IgnoreName) if err != nil { - fmt.Println(err) - return false + return false, err } m.context.ForceUpdateDmg = cli.Force m.context.UpdateIndex = cli.Update || cli.AddOnly @@ -182,7 +182,23 @@ func (m *Main) process() bool { m.context.SkipSymlinks = cli.SkipSymlinks m.context.SkipSubdirectories = cli.NoRecurse m.context.TrackDirectories = !cli.NoDirInIndex - m.context.UseSingleDb = cli.IndexDb + + pathList := cli.Paths + if cli.Db { + var root string + root, pathList, err = m.context.UseIndexDb(pathList) + if err == nil { + // pathList is relative to root + err = os.Chdir(root) + if m.progress != Quiet { + fmt.Println("Using indexdb in " + root) + } + m.log("using indexdb in " + root) + } + if err != nil { + return false, err + } + } var wg sync.WaitGroup wg.Add(1) @@ -190,13 +206,13 @@ func (m *Main) process() bool { defer wg.Done() m.showStatus() }() - m.context.Start(cli.Paths) + m.context.Start(pathList) wg.Wait() - return true + return true, nil } -func (m *Main) printResult() { +func (m *Main) printResult() error { cprint := func(col, text string) { if m.progress != Quiet { if m.progress == Fancy { @@ -278,11 +294,12 @@ func (m *Main) printResult() { } if len(m.dmgList) > 0 || len(m.errList) > 0 { - os.Exit(1) + return errors.New("fail") } + return nil } -func (m *Main) run() { +func (m *Main) run() int { if len(os.Args) < 2 { os.Args = append(os.Args, "--help") @@ -303,13 +320,25 @@ func (m *Main) run() { if cli.Tips { fmt.Println(strings.ReplaceAll(helpTips, "", configPath)) - os.Exit(0) + return 0 } if cli.Version { fmt.Println("github.com/laktak/chkbit") fmt.Println(appVersion) - return + return 0 + } + + if cli.InitDb { + if len(cli.Paths) != 1 { + fmt.Println("error: specify a path") + return 1 + } + if err := chkbit.InitializeIndexDb(cli.Paths[0], cli.Force); err != nil { + fmt.Println("error: " + err.Error()) + return 1 + } + return 0 } m.verbose = cli.Verbose || cli.ShowIgnoredOnly @@ -317,8 +346,8 @@ func (m *Main) run() { m.logVerbose = cli.LogVerbose f, err := os.OpenFile(cli.LogFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err != nil { - fmt.Println(err) - return + fmt.Println("error: " + err.Error()) + return 1 } defer f.Close() m.logger = log.New(f, "", 0) @@ -335,37 +364,29 @@ func (m *Main) run() { } if len(cli.Paths) > 0 { - if cli.IndexDb { - if len(cli.Paths) > 1 { - fmt.Println("Only one path allowed with --index-db switch") - os.Exit(1) - } + m.log("chkbit " + strings.Join(cli.Paths, ", ")) - mainPath := cli.Paths[0] - err := os.Chdir(mainPath) - if err != nil { - fmt.Println(err) - os.Exit(1) + if showRes, err := m.process(); err == nil { + if showRes && !cli.ShowIgnoredOnly { + if m.printResult() != nil { + return 1 + } } - cli.Paths[0] = "." - m.log("chkbit at " + mainPath) - } else { - m.log("chkbit " + strings.Join(cli.Paths, ", ")) - } - - if m.process() && !cli.ShowIgnoredOnly { - m.printResult() + fmt.Println("error: " + err.Error()) + return 1 } } else { - fmt.Println("specify a path to check, see -h") + fmt.Println("error: specify a path, see -h") } + return 0 } func main() { defer func() { if r := recover(); r != nil { + // panic fmt.Println(r) os.Exit(1) } @@ -378,5 +399,5 @@ func main() { fps: util.NewRateCalc(time.Second, (termWidth-70)/2), bps: util.NewRateCalc(time.Second, (termWidth-70)/2), } - m.run() + os.Exit(m.run()) } diff --git a/context.go b/context.go index 9016b73..79f018e 100644 --- a/context.go +++ b/context.go @@ -2,8 +2,10 @@ package chkbit import ( "errors" + "fmt" "os" "path/filepath" + "strings" "sync" ) @@ -22,13 +24,11 @@ type Context struct { IndexFilename string IgnoreFilename string - UseSingleDb bool - WorkQueue chan *WorkItem LogQueue chan *LogEvent PerfQueue chan *PerfEvent - db *indexDb + store *store mutex sync.Mutex NumTotal int @@ -56,7 +56,7 @@ func NewContext(numWorkers int, hashAlgo string, indexFilename string, ignoreFil WorkQueue: make(chan *WorkItem, numWorkers*10), LogQueue: make(chan *LogEvent, numWorkers*100), PerfQueue: make(chan *PerfEvent, numWorkers*10), - db: &indexDb{}, + store: &store{}, }, nil } @@ -121,13 +121,14 @@ func (context *Context) Start(pathList []string) { context.NumNew = 0 context.NumUpd = 0 context.NumDel = 0 - err := context.db.Open(context.UseSingleDb, !context.UpdateIndex) + + err := context.store.Open(!context.UpdateIndex) if err != nil { - context.logErr(context.db.GetDbPath(), err) + context.logErr(indexDbName, err) context.LogQueue <- nil return } - defer context.db.Close() + defer context.store.Close() var wg sync.WaitGroup wg.Add(context.NumWorkers) @@ -200,3 +201,35 @@ func (context *Context) scanDir(root string, parentIgnore *Ignore) { } } } + +func (context *Context) UseIndexDb(pathList []string) (root string, relativePathList []string, err error) { + + if len(pathList) == 0 { + return "", nil, errors.New("missing path(s)") + } + err = context.store.UseDb(pathList[0]) + if err == nil { + + root = context.store.dbPath + + for _, path := range pathList { + path, err = filepath.Abs(path) + if err != nil { + return "", nil, err + } + + // below root? + if !strings.HasPrefix(path, root) { + return "", nil, fmt.Errorf("path %s is not below the indexdb in %s", path, root) + } + + relativePath, err := filepath.Rel(root, path) + if err != nil { + return "", nil, err + } + relativePathList = append(relativePathList, relativePath) + } + } + + return +} diff --git a/index.go b/index.go index d75f626..f16ca01 100644 --- a/index.go +++ b/index.go @@ -246,7 +246,7 @@ func (i *Index) save() (bool, error) { return false, err } - err = i.context.db.Save(i.getIndexFilepath(), file) + err = i.context.store.Save(i.getIndexFilepath(), file) if err != nil { return false, err } @@ -259,7 +259,7 @@ func (i *Index) save() (bool, error) { } func (i *Index) load() error { - file, err := i.context.db.Load(i.getIndexFilepath()) + file, err := i.context.store.Load(i.getIndexFilepath()) if file == nil || err != nil { return err } diff --git a/indexdb.go b/indexdb.go deleted file mode 100644 index 02e2497..0000000 --- a/indexdb.go +++ /dev/null @@ -1,95 +0,0 @@ -package chkbit - -import ( - "os" - "path/filepath" - - bolt "go.etcd.io/bbolt" -) - -type indexDb struct { - useSingleDb bool - conn *bolt.DB -} - -func (db *indexDb) GetDbPath() string { - return ".chkbitdb" -} - -func (db *indexDb) Open(useSingleDb, readOnly bool) error { - var err error - db.useSingleDb = useSingleDb - if useSingleDb { - opt := &bolt.Options{ - ReadOnly: readOnly, - Timeout: 0, - NoGrowSync: false, - FreelistType: bolt.FreelistArrayType, - } - if readOnly { - _, err := os.Stat(db.GetDbPath()) - if os.IsNotExist(err) { - return nil - } - } - // todo: write to new db - db.conn, err = bolt.Open(db.GetDbPath(), 0600, opt) - if err == nil && !readOnly { - err = db.conn.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucketIfNotExists([]byte("data")) - return err - }) - } - } - return err -} - -func (db *indexDb) Close() { - if db.useSingleDb && db.conn != nil { - db.conn.Close() - } -} - -func (db *indexDb) Load(indexPath string) ([]byte, error) { - var err error - var value []byte - if db.useSingleDb { - if db.conn == nil { - // readOnly without db - return nil, nil - } - err = db.conn.View(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("data")) - value = b.Get([]byte(indexPath)) - return nil - }) - } else { - if _, err = os.Stat(indexPath); err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, err - } - value, err = os.ReadFile(indexPath) - } - return value, err -} - -func (db *indexDb) Save(indexPath string, value []byte) error { - var err error - if db.useSingleDb { - err = db.conn.Update(func(tx *bolt.Tx) error { - b := tx.Bucket([]byte("data")) - return b.Put([]byte(indexPath), value) - }) - } else { - // try to preserve the directory mod time but ignore if unsupported - dirPath := filepath.Dir(indexPath) - dirStat, dirErr := os.Stat(dirPath) - err = os.WriteFile(indexPath, value, 0644) - if dirErr == nil { - os.Chtimes(dirPath, dirStat.ModTime(), dirStat.ModTime()) - } - } - return err -} diff --git a/store.go b/store.go new file mode 100644 index 0000000..c2c4750 --- /dev/null +++ b/store.go @@ -0,0 +1,149 @@ +package chkbit + +import ( + "errors" + "os" + "path/filepath" + + bolt "go.etcd.io/bbolt" +) + +type store struct { + useDb bool + dbPath string + conn *bolt.DB +} + +const ( + indexDbName = ".chkbitdb" +) + +func (s *store) UseDb(path string) error { + var err error + s.dbPath, err = LocateIndexDb(path) + if err == nil { + s.useDb = true + } + return err +} + +func (s *store) GetDbFile() string { + return filepath.Join(s.dbPath, indexDbName) +} + +func (s *store) Open(readOnly bool) error { + var err error + if s.useDb { + opt := &bolt.Options{ + ReadOnly: readOnly, + Timeout: 0, + NoGrowSync: false, + FreelistType: bolt.FreelistArrayType, + } + _, err = os.Stat(s.GetDbFile()) + if os.IsNotExist(err) { + return nil + } + s.conn, err = bolt.Open(s.GetDbFile(), 0600, opt) + if err == nil && !readOnly { + err = s.conn.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte("data")) + return err + }) + } + } + return err +} + +func (s *store) Close() { + if s.useDb && s.conn != nil { + s.conn.Close() + } +} + +func (s *store) Load(indexPath string) ([]byte, error) { + var err error + var value []byte + if s.useDb { + if s.conn == nil { + // readOnly without db + return nil, nil + } + err = s.conn.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("data")) + value = b.Get([]byte(indexPath)) + return nil + }) + } else { + if _, err = os.Stat(indexPath); err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + value, err = os.ReadFile(indexPath) + } + return value, err +} + +func (s *store) Save(indexPath string, value []byte) error { + var err error + if s.useDb { + err = s.conn.Update(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte("data")) + return b.Put([]byte(indexPath), value) + }) + } else { + // try to preserve the directory mod time but ignore if unsupported + dirPath := filepath.Dir(indexPath) + dirStat, dirErr := os.Stat(dirPath) + err = os.WriteFile(indexPath, value, 0644) + if dirErr == nil { + os.Chtimes(dirPath, dirStat.ModTime(), dirStat.ModTime()) + } + } + return err +} + +func InitializeIndexDb(path string, force bool) error { + file := filepath.Join(path, indexDbName) + _, err := os.Stat(file) + if !os.IsNotExist(err) { + if force { + err := os.Remove(file) + if err != nil { + return err + } + } else { + return errors.New("file exists") + } + + } + conn, err := bolt.Open(file, 0600, nil) + if err != nil { + return err + } + return conn.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte("data")) + return err + }) +} + +func LocateIndexDb(path string) (string, error) { + var err error + if path, err = filepath.Abs(path); err != nil { + return "", err + } + for { + file := filepath.Join(path, indexDbName) + _, err = os.Stat(file) + if !os.IsNotExist(err) { + return path, nil + } + path = filepath.Dir(path) + if len(path) < 1 || path[len(path)-1] == filepath.Separator { + // reached root + return "", errors.New("index db could not be located (forgot to initialize?)") + } + } +}