123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296 |
- package main
- import (
- "bufio"
- "crypto/sha256"
- "encoding/hex"
- "encoding/json"
- "fmt"
- "io"
- "io/ioutil"
- "log"
- "os"
- "path/filepath"
- "runtime"
- "sync"
- "time"
- flag "github.com/spf13/pflag"
- )
- // Fdhashes struct for holding all informations about one folder.
- type Fdhashes struct {
- Path string
- Hashes map[string]string
- Times map[string]time.Time
- Dirty bool
- }
- var hashes map[string]Fdhashes
- var wg sync.WaitGroup
- var mu sync.RWMutex
- var driveLetter string
- var rewrite bool
- var compare bool
- func init() {
- flag.BoolVarP(&rewrite, "rewrite", "r", false, "rewrite all fhhashes files.")
- flag.BoolVarP(&compare, "compare", "c", false, "compare all file hashes and writing a compartion report.")
- }
- func main() {
- log.Println("starting GoHash")
- runtime.GOMAXPROCS(5)
- hashes = make(map[string]Fdhashes)
- flag.Parse()
- myFile := flag.Arg(0)
- file, err := os.Stat(myFile)
- if os.IsNotExist(err) {
- log.Fatalln("File does not exists:", myFile)
- }
- if file.IsDir() {
- log.Println("start with folder:", myFile)
- driveLetter = ""
- if runtime.GOOS == "windows" {
- driveLetter = filepath.VolumeName(myFile) + "/"
- }
- if compare {
- compareFolder(myFile)
- } else {
- processFolder(myFile)
- fmt.Println("waiting")
- wg.Wait()
- saveAllHashFiles()
- }
- } else {
- log.Printf("file %s has hash %s\n", myFile, getSha256Hash(myFile))
- }
- log.Println("done")
- }
- func getSha256Hash(fileStr string) string {
- f, err := os.Open(fileStr)
- if err != nil {
- log.Fatal(err)
- }
- defer f.Close()
- h := sha256.New()
- if _, err := io.Copy(h, f); err != nil {
- log.Fatal(err)
- }
- return hex.EncodeToString(h.Sum(nil))
- }
- var lock1 = sync.RWMutex{}
- var lock2 = sync.RWMutex{}
- func outputHash(fileStr string) {
- var hashFile Fdhashes
- doHash := true
- defer wg.Done()
- dir, fileName := filepath.Split(fileStr)
- if fileName == ".fdhashes3" {
- return
- }
- // checking if hash is present
- mu.Lock()
- hashFile, ok := hashes[dir]
- if !ok {
- _, err := os.Stat(dir + ".fdhashes3")
- if os.IsNotExist(err) {
- hashFile = Fdhashes{Path: dir, Hashes: make(map[string]string), Times: make(map[string]time.Time), Dirty: true}
- } else {
- hashFile = loadHashfile(dir + ".fdhashes3")
- }
- hashes[dir] = hashFile
- }
- lock1.RLock()
- _, ok = hashFile.Hashes[fileName]
- lock1.RUnlock()
- mu.Unlock()
- doHash = !ok
- // checking if dattime is identically
- file, _ := os.Stat(fileStr)
- time := file.ModTime()
- lock2.RLock()
- savedTime, ok := hashFile.Times[fileName]
- lock2.RUnlock()
- if !time.Equal(savedTime) || !ok {
- doHash = true
- }
- if doHash {
- hash := getSha256Hash(fileStr)
- mu.Lock()
- lock1.Lock()
- hashFile.Hashes[fileName] = hash
- lock1.Unlock()
- lock2.Lock()
- hashFile.Times[fileName] = time
- lock2.Unlock()
- saveHashfile(&hashFile)
- hashes[dir] = hashFile
- mu.Unlock()
- log.Printf("file \"%s\" has hash \"%s\"\n", fileStr, hash)
- }
- }
- var count int
- var addWork int
- func processFolder(folder string) {
- count = 0
- addWork = 0
- err := filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
- count++
- if (count % 100) == 0 {
- fmt.Print(".")
- }
- if (count % 10000) == 0 {
- fmt.Println()
- }
- filename := info.Name()
- if filename[0:1] != "." {
- if info.IsDir() {
- fmt.Println(path)
- }
- if !info.IsDir() {
- addWork++
- wg.Add(1)
- go outputHash(path)
- if addWork > 1000 {
- fmt.Println("x")
- wg.Wait()
- saveAllHashFiles()
- addWork = 0
- }
- }
- }
- return nil
- })
- if err != nil {
- panic(err)
- }
- }
- func saveHashfile(hashFile *Fdhashes) {
- hashFile.Dirty = true
- }
- func saveAllHashFiles() {
- hashList := make([]Fdhashes, 0)
- for _, hashFile := range hashes {
- if hashFile.Dirty {
- hashFile.Dirty = false
- b, err := json.Marshal(hashFile)
- if err != nil {
- fmt.Println(err)
- return
- }
- err = ioutil.WriteFile(hashFile.Path+".fdhashes3", b, 0644)
- if err != nil {
- panic(err)
- }
- hashList = append(hashList, hashFile)
- }
- }
- hashes = make(map[string]Fdhashes)
- for _, hashFile := range hashList {
- hashes[hashFile.Path] = hashFile
- }
- }
- func loadHashfile(fileStr string) Fdhashes {
- dir, _ := filepath.Split(fileStr)
- data := Fdhashes{Path: dir, Hashes: make(map[string]string), Times: make(map[string]time.Time), Dirty: false}
- if !rewrite {
- file, err := ioutil.ReadFile(fileStr)
- if err != nil {
- panic(err)
- }
- err = json.Unmarshal([]byte(file), &data)
- if err != nil {
- log.Printf("can't read file %s", fileStr)
- }
- }
- if data.Path != dir {
- data.Path = dir
- data.Dirty = true
- }
- return data
- }
- func compareFolder(folder string) {
- loadAllHashFiles(folder)
- size := len(hashes)
- f, err := os.Create("report.txt")
- check(err)
- w := bufio.NewWriter(f)
- count := 0
- for _, hashFile := range hashes {
- count++
- fmt.Printf("%d (%d) checking: %s\n", count, size, hashFile.Path)
- // fmt.Printf("checking: %s\n", hashFile.Path)
- for filename, hash := range hashFile.Hashes {
- if value, found := search(hash, filename, hashFile.Path); found {
- w.WriteString("found identically hash\n")
- w.WriteString(fmt.Sprintf(" src: %s/%s\n", hashFile.Path, filename))
- w.WriteString(fmt.Sprintf(" dest: %s\n", value))
- w.Flush()
- }
- }
- }
- }
- func search(srcHash string, exFilename string, exFilepath string) (value string, found bool) {
- for _, hashFile := range hashes {
- for filename, hash := range hashFile.Hashes {
- if (filename != exFilename) && (hashFile.Path != exFilepath) {
- if hash == srcHash {
- value += fmt.Sprintf("%s/%s;", hashFile.Path, filename)
- found = true
- }
- }
- }
- }
- return
- }
- func loadAllHashFiles(folder string) {
- count = 0
- addWork = 0
- err := filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
- //filename := info.Name()
- if info.IsDir() {
- fmt.Print(".")
- hashFile, ok := hashes[path]
- if !ok {
- _, err := os.Stat(path + "/.fdhashes3")
- if os.IsNotExist(err) {
- hashFile = Fdhashes{Path: path, Hashes: make(map[string]string), Times: make(map[string]time.Time), Dirty: true}
- } else {
- hashFile = loadHashfile(path + "/.fdhashes3")
- }
- hashes[path] = hashFile
- }
- }
- return nil
- })
- if err != nil {
- panic(err)
- }
- fmt.Printf("\nfound %d hash files.\n", len(hashes))
- }
- func check(e error) {
- if e != nil {
- panic(e)
- }
- }
|