store index in db #22

laktak · Dec 2, 2024 · 7329c24 · 7329c24
1 parent cc03b8c
commit 7329c24
Show file tree

Hide file tree

Showing 6 changed files with 270 additions and 152 deletions.
diff --git a/README.md b/README.md
@@ -111,20 +111,22 @@ Flags:
       --ignore-name=".chkbitignore"
                                 filename that chkbit reads its ignore list from,
                                 needs to start with '.' (default: .chkbitignore)
-      --index-db                use a index database instead of index files
+      --db                      use a index database instead of index files
   -w, --workers=5               number of workers to use (default: 5)
       --[no-]plain              show plain status instead of being fancy
   -q, --[no-]quiet              quiet, don't show progress/information
   -v, --[no-]verbose            verbose output
   -V, --version                 show version information
 
-mode
-  -c, --check                check mode: chkbit will verify files in readonly
-                             mode (default mode)
-  -u, --update               update mode: add and update indices
-  -a, --add-only             add mode: only add new and modified files, do not
-                             check existing (quicker)
-  -i, --show-ignored-only    show-ignored mode: only show ignored files
+Mode
+  -c, --check                chkbit will verify files in readonly mode (default
+                             mode)
+  -u, --update               add and update indices
+  -a, --add-only             only add new and modified files, do not check
+                             existing (quicker)
+      --init-db              initialize a new index database at the given path
+                             for use with --db
+  -i, --show-ignored-only    only show ignored files
 ```
 
 ```
@@ -188,6 +190,14 @@ Add a `.chkbitignore` file containing the names of the files/directories you wis
 - you can use `path/sub/name` to ignore a file/directory in a sub path
 - hidden files (starting with a `.`) are ignored by default unless you use the `-d` option
 
+## Database Usage
+
+*experimental*
+
+To use a chkbit database you need to initalize it first with `chkbit --init-db PATH`.
+
+Then you can run `chkbit` on anything below PATH and it will be tracked in the database.
+
 
 ## chkbit as a Go module
 

diff --git a/cmd/chkbit/main.go b/cmd/chkbit/main.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -45,10 +46,11 @@ var (
 var cli struct {
 	Paths           []string `arg:"" optional:"" name:"paths" help:"directories to check"`
 	Tips            bool     `short:"H" help:"Show tips."`
-	Check           bool     `short:"c" help:"check mode: chkbit will verify files in readonly mode (default mode)" xor:"mode" group:"mode"`
-	Update          bool     `short:"u" help:"update mode: add and update indices" xor:"mode" group:"mode"`
-	AddOnly         bool     `short:"a" help:"add mode: only add new and modified files, do not check existing (quicker)" xor:"mode" group:"mode"`
-	ShowIgnoredOnly bool     `short:"i" help:"show-ignored mode: only show ignored files" xor:"mode" group:"mode"`
+	Check           bool     `short:"c" help:"chkbit will verify files in readonly mode (default mode)" xor:"mode" group:"Mode"`
+	Update          bool     `short:"u" help:"add and update indices" xor:"mode" group:"Mode"`
+	AddOnly         bool     `short:"a" help:"only add new and modified files, do not check existing (quicker)" xor:"mode" group:"Mode"`
+	InitDb          bool     `help:"initialize a new index database at the given path for use with --db" xor:"mode" group:"Mode"`
+	ShowIgnoredOnly bool     `short:"i" help:"only show ignored files" xor:"mode" group:"Mode"`
 	ShowMissing     bool     `short:"m" help:"show missing files/directories" negatable:""`
 	IncludeDot      bool     `short:"d" help:"include dot files" negatable:""`
 	SkipSymlinks    bool     `short:"S" help:"do not follow symlinks" negatable:""`
@@ -60,7 +62,7 @@ var cli struct {
 	Algo            string   `default:"blake3" help:"hash algorithm: md5, sha512, blake3 (default: blake3)"`
 	IndexName       string   `default:".chkbit" help:"filename where chkbit stores its hashes, needs to start with '.' (default: .chkbit)"`
 	IgnoreName      string   `default:".chkbitignore" help:"filename that chkbit reads its ignore list from, needs to start with '.' (default: .chkbitignore)"`
-	IndexDb         bool     `help:"use a index database instead of index files"`
+	Db              bool     `help:"use a index database instead of index files"`
 	Workers         int      `short:"w" default:"5" help:"number of workers to use (default: 5)"`
 	Plain           bool     `help:"show plain status instead of being fancy" negatable:""`
 	Quiet           bool     `short:"q" help:"quiet, don't show progress/information" negatable:""`
@@ -159,19 +161,17 @@ func (m *Main) showStatus() {
 	}
 }
 
-func (m *Main) process() bool {
+func (m *Main) process() (bool, error) {
 	// verify mode
 	var b01 = map[bool]int8{false: 0, true: 1}
 	if b01[cli.Check]+b01[cli.Update]+b01[cli.AddOnly]+b01[cli.ShowIgnoredOnly] > 1 {
-		fmt.Println("Error: can only run one mode at a time!")
-		os.Exit(1)
+		return false, errors.New("can only run one mode at a time")
 	}
 
 	var err error
 	m.context, err = chkbit.NewContext(cli.Workers, cli.Algo, cli.IndexName, cli.IgnoreName)
 	if err != nil {
-		fmt.Println(err)
-		return false
+		return false, err
 	}
 	m.context.ForceUpdateDmg = cli.Force
 	m.context.UpdateIndex = cli.Update || cli.AddOnly
@@ -182,21 +182,37 @@ func (m *Main) process() bool {
 	m.context.SkipSymlinks = cli.SkipSymlinks
 	m.context.SkipSubdirectories = cli.NoRecurse
 	m.context.TrackDirectories = !cli.NoDirInIndex
-	m.context.UseSingleDb = cli.IndexDb
+
+	pathList := cli.Paths
+	if cli.Db {
+		var root string
+		root, pathList, err = m.context.UseIndexDb(pathList)
+		if err == nil {
+			// pathList is relative to root
+			err = os.Chdir(root)
+			if m.progress != Quiet {
+				fmt.Println("Using indexdb in " + root)
+			}
+			m.log("using indexdb in " + root)
+		}
+		if err != nil {
+			return false, err
+		}
+	}
 
 	var wg sync.WaitGroup
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
 		m.showStatus()
 	}()
-	m.context.Start(cli.Paths)
+	m.context.Start(pathList)
 	wg.Wait()
 
-	return true
+	return true, nil
 }
 
-func (m *Main) printResult() {
+func (m *Main) printResult() error {
 	cprint := func(col, text string) {
 		if m.progress != Quiet {
 			if m.progress == Fancy {
@@ -278,11 +294,12 @@ func (m *Main) printResult() {
 	}
 
 	if len(m.dmgList) > 0 || len(m.errList) > 0 {
-		os.Exit(1)
+		return errors.New("fail")
 	}
+	return nil
 }
 
-func (m *Main) run() {
+func (m *Main) run() int {
 
 	if len(os.Args) < 2 {
 		os.Args = append(os.Args, "--help")
@@ -303,22 +320,34 @@ func (m *Main) run() {
 
 	if cli.Tips {
 		fmt.Println(strings.ReplaceAll(helpTips, "<config-file>", configPath))
-		os.Exit(0)
+		return 0
 	}
 
 	if cli.Version {
 		fmt.Println("github.com/laktak/chkbit")
 		fmt.Println(appVersion)
-		return
+		return 0
+	}
+
+	if cli.InitDb {
+		if len(cli.Paths) != 1 {
+			fmt.Println("error: specify a path")
+			return 1
+		}
+		if err := chkbit.InitializeIndexDb(cli.Paths[0], cli.Force); err != nil {
+			fmt.Println("error: " + err.Error())
+			return 1
+		}
+		return 0
 	}
 
 	m.verbose = cli.Verbose || cli.ShowIgnoredOnly
 	if cli.LogFile != "" {
 		m.logVerbose = cli.LogVerbose
 		f, err := os.OpenFile(cli.LogFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
 		if err != nil {
-			fmt.Println(err)
-			return
+			fmt.Println("error: " + err.Error())
+			return 1
 		}
 		defer f.Close()
 		m.logger = log.New(f, "", 0)
@@ -335,37 +364,29 @@ func (m *Main) run() {
 	}
 
 	if len(cli.Paths) > 0 {
-		if cli.IndexDb {
-			if len(cli.Paths) > 1 {
-				fmt.Println("Only one path allowed with --index-db switch")
-				os.Exit(1)
-			}
+		m.log("chkbit " + strings.Join(cli.Paths, ", "))
 
-			mainPath := cli.Paths[0]
-			err := os.Chdir(mainPath)
-			if err != nil {
-				fmt.Println(err)
-				os.Exit(1)
+		if showRes, err := m.process(); err == nil {
+			if showRes && !cli.ShowIgnoredOnly {
+				if m.printResult() != nil {
+					return 1
+				}
 			}
-			cli.Paths[0] = "."
-			m.log("chkbit at " + mainPath)
-
 		} else {
-			m.log("chkbit " + strings.Join(cli.Paths, ", "))
-		}
-
-		if m.process() && !cli.ShowIgnoredOnly {
-			m.printResult()
+			fmt.Println("error: " + err.Error())
+			return 1
 		}
 
 	} else {
-		fmt.Println("specify a path to check, see -h")
+		fmt.Println("error: specify a path, see -h")
 	}
+	return 0
 }
 
 func main() {
 	defer func() {
 		if r := recover(); r != nil {
+			// panic
 			fmt.Println(r)
 			os.Exit(1)
 		}
@@ -378,5 +399,5 @@ func main() {
 		fps:       util.NewRateCalc(time.Second, (termWidth-70)/2),
 		bps:       util.NewRateCalc(time.Second, (termWidth-70)/2),
 	}
-	m.run()
+	os.Exit(m.run())
 }
diff --git a/context.go b/context.go
@@ -2,8 +2,10 @@ package chkbit
 
 import (
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 )
 
@@ -22,13 +24,11 @@ type Context struct {
 	IndexFilename      string
 	IgnoreFilename     string
 
-	UseSingleDb bool
-
 	WorkQueue chan *WorkItem
 	LogQueue  chan *LogEvent
 	PerfQueue chan *PerfEvent
 
-	db *indexDb
+	store *store
 
 	mutex     sync.Mutex
 	NumTotal  int
@@ -56,7 +56,7 @@ func NewContext(numWorkers int, hashAlgo string, indexFilename string, ignoreFil
 		WorkQueue:      make(chan *WorkItem, numWorkers*10),
 		LogQueue:       make(chan *LogEvent, numWorkers*100),
 		PerfQueue:      make(chan *PerfEvent, numWorkers*10),
-		db:             &indexDb{},
+		store:          &store{},
 	}, nil
 }
 
@@ -121,13 +121,14 @@ func (context *Context) Start(pathList []string) {
 	context.NumNew = 0
 	context.NumUpd = 0
 	context.NumDel = 0
-	err := context.db.Open(context.UseSingleDb, !context.UpdateIndex)
+
+	err := context.store.Open(!context.UpdateIndex)
 	if err != nil {
-		context.logErr(context.db.GetDbPath(), err)
+		context.logErr(indexDbName, err)
 		context.LogQueue <- nil
 		return
 	}
-	defer context.db.Close()
+	defer context.store.Close()
 
 	var wg sync.WaitGroup
 	wg.Add(context.NumWorkers)
@@ -200,3 +201,35 @@ func (context *Context) scanDir(root string, parentIgnore *Ignore) {
 		}
 	}
 }
+
+func (context *Context) UseIndexDb(pathList []string) (root string, relativePathList []string, err error) {
+
+	if len(pathList) == 0 {
+		return "", nil, errors.New("missing path(s)")
+	}
+	err = context.store.UseDb(pathList[0])
+	if err == nil {
+
+		root = context.store.dbPath
+
+		for _, path := range pathList {
+			path, err = filepath.Abs(path)
+			if err != nil {
+				return "", nil, err
+			}
+
+			// below root?
+			if !strings.HasPrefix(path, root) {
+				return "", nil, fmt.Errorf("path %s is not below the indexdb in %s", path, root)
+			}
+
+			relativePath, err := filepath.Rel(root, path)
+			if err != nil {
+				return "", nil, err
+			}
+			relativePathList = append(relativePathList, relativePath)
+		}
+	}
+
+	return
+}
diff --git a/index.go b/index.go
@@ -246,7 +246,7 @@ func (i *Index) save() (bool, error) {
 			return false, err
 		}
 
-		err = i.context.db.Save(i.getIndexFilepath(), file)
+		err = i.context.store.Save(i.getIndexFilepath(), file)
 		if err != nil {
 			return false, err
 		}
@@ -259,7 +259,7 @@ func (i *Index) save() (bool, error) {
 }
 
 func (i *Index) load() error {
-	file, err := i.context.db.Load(i.getIndexFilepath())
+	file, err := i.context.store.Load(i.getIndexFilepath())
 	if file == nil || err != nil {
 		return err
 	}