Sfoglia il codice sorgente

faster report generation

Klaas, Wilfried 5 anni fa
parent
commit
38166cdf64
4 ha cambiato i file con 54 aggiunte e 1 eliminazioni
  1. 1 0
      .gitignore
  2. 47 0
      GoHash.go
  3. 4 1
      go.mod
  4. 2 0
      go.sum

+ 1 - 0
.gitignore

@@ -3,3 +3,4 @@
 pkg
 src/github.com
 __debug_bin
+report.txt

+ 47 - 0
GoHash.go

@@ -15,6 +15,8 @@ import (
 	"sync"
 	"time"
 
+	"code.cloudfoundry.org/bytefmt"
+
 	flag "github.com/spf13/pflag"
 )
 
@@ -229,6 +231,51 @@ func loadHashfile(fileStr string) Fdhashes {
 
 func compareFolder(folder string) {
 	loadAllHashFiles(folder)
+
+	index := make(map[string][]string)
+
+	for _, hashFile := range hashes {
+		for filename, hash := range hashFile.Hashes {
+			values := index[hash]
+			if values == nil {
+				values = make([]string, 0)
+			}
+			values = append(values, fmt.Sprintf("%s/%s", hashFile.Path, filename))
+			index[hash] = values
+		}
+	}
+
+	size := len(index)
+	f, err := os.Create("report.txt")
+	check(err)
+	w := bufio.NewWriter(f)
+	count := 0
+	var filesize int64
+	fileCount := 0
+	for _, values := range index {
+		count++
+		if count%100 == 0 {
+			fmt.Printf("%d (%d) checking\n", count, size)
+		}
+		if len(values) > 1 {
+			info, err := os.Stat(values[0])
+			if err == nil {
+				w.WriteString(fmt.Sprintf("found identically hash: size: %d\n", info.Size()))
+				filesize += int64(len(values)-1) * info.Size()
+			}
+			fileCount += len(values) - 1
+			for _, filename := range values {
+				w.WriteString(fmt.Sprintf("  %s\n", filename))
+			}
+			w.Flush()
+		}
+	}
+	w.WriteString(fmt.Sprintf("can save up to %s on %d files\n", bytefmt.ByteSize(uint64(filesize)), fileCount))
+	w.Flush()
+}
+
+func compareFolder2(folder string) {
+	loadAllHashFiles(folder)
 	size := len(hashes)
 	f, err := os.Create("report.txt")
 	check(err)

+ 4 - 1
go.mod

@@ -2,4 +2,7 @@ module github.com/willie/gohash
 
 go 1.12
 
-require github.com/spf13/pflag v1.0.3
+require (
+	code.cloudfoundry.org/bytefmt v0.0.0-20190819182555-854d396b647c
+	github.com/spf13/pflag v1.0.3
+)

+ 2 - 0
go.sum

@@ -1,2 +1,4 @@
+code.cloudfoundry.org/bytefmt v0.0.0-20190819182555-854d396b647c h1:2RuXx1+tSNWRjxhY0Bx52kjV2odJQ0a6MTbfTPhGAkg=
+code.cloudfoundry.org/bytefmt v0.0.0-20190819182555-854d396b647c/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=