GoHash.go 3.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166
  1. package main
  2. import (
  3. "crypto/sha256"
  4. "encoding/hex"
  5. "encoding/json"
  6. "flag"
  7. "fmt"
  8. "io"
  9. "io/ioutil"
  10. "log"
  11. "os"
  12. "path/filepath"
  13. "runtime"
  14. "sync"
  15. )
  16. // Fdhashes struct for holding all informations about one folder.
  17. type Fdhashes struct {
  18. Hashes map[string]string
  19. }
  20. var hashes map[string]Fdhashes
  21. var wg sync.WaitGroup
  22. var mu sync.RWMutex
  23. var driveLetter string
  24. var rewrite bool
  25. func init() {
  26. flag.BoolVar(&rewrite, "rewrite", false, "rewrite all fhhashes files.")
  27. }
  28. func main() {
  29. log.Println("starting GoHash")
  30. runtime.GOMAXPROCS(5)
  31. hashes = make(map[string]Fdhashes)
  32. flag.Parse()
  33. myFile := flag.Arg(0)
  34. file, err := os.Stat(myFile)
  35. if os.IsNotExist(err) {
  36. log.Fatalln("File does not exists:", myFile)
  37. }
  38. if file.IsDir() {
  39. log.Println("start with folder:", myFile)
  40. driveLetter = ""
  41. if runtime.GOOS == "windows" {
  42. driveLetter = filepath.VolumeName(myFile) + "/"
  43. }
  44. processFolder(myFile)
  45. } else {
  46. log.Printf("file %s has hash %s\n", myFile, getSha256Hash(myFile))
  47. }
  48. fmt.Println("waiting")
  49. wg.Wait()
  50. log.Println("done")
  51. }
  52. func getSha256Hash(fileStr string) string {
  53. f, err := os.Open(fileStr)
  54. if err != nil {
  55. log.Fatal(err)
  56. }
  57. defer f.Close()
  58. h := sha256.New()
  59. if _, err := io.Copy(h, f); err != nil {
  60. log.Fatal(err)
  61. }
  62. return hex.EncodeToString(h.Sum(nil))
  63. }
  64. func outputHash(fileStr string) {
  65. var hashFile Fdhashes
  66. doHash := true
  67. defer wg.Done()
  68. dir, fileName := filepath.Split(fileStr)
  69. if fileName == ".fdhashes3" {
  70. return
  71. }
  72. mu.Lock()
  73. hashFile, ok := hashes[dir]
  74. if !ok {
  75. _, err := os.Stat(dir + ".fdhashes3")
  76. if os.IsNotExist(err) {
  77. hashFile = Fdhashes{Hashes: make(map[string]string)}
  78. } else {
  79. hashFile = loadHashfile(dir + ".fdhashes3")
  80. }
  81. hashes[dir] = hashFile
  82. saveHashfile(hashFile, dir)
  83. }
  84. _, ok = hashFile.Hashes[fileName]
  85. doHash = !ok
  86. mu.Unlock()
  87. if doHash {
  88. hash := getSha256Hash(fileStr)
  89. mu.Lock()
  90. hashFile.Hashes[fileName] = hash
  91. hashes[dir] = hashFile
  92. saveHashfile(hashFile)
  93. mu.Unlock()
  94. log.Printf("file \"%s\" has hash \"%s\"\n", fileStr, hash)
  95. }
  96. }
  97. var count int
  98. func processFolder(folder string) {
  99. count = 0
  100. err := filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
  101. count++
  102. if (count % 100) == 0 {
  103. fmt.Print(".")
  104. }
  105. if (count % 10000) == 0 {
  106. fmt.Println()
  107. }
  108. filename := info.Name()
  109. if filename[0:1] != "." {
  110. if info.IsDir() {
  111. fmt.Println(path)
  112. }
  113. if !info.IsDir() {
  114. wg.Add(1)
  115. go outputHash(path)
  116. }
  117. }
  118. return nil
  119. })
  120. if err != nil {
  121. panic(err)
  122. }
  123. }
  124. func saveHashfile(hashFile Fdhashes, dir string) {
  125. b, err := json.Marshal(hashFile)
  126. if err != nil {
  127. fmt.Println(err)
  128. return
  129. }
  130. err = ioutil.WriteFile(dir+".fdhashes3", b, 0644)
  131. if err != nil {
  132. panic(err)
  133. }
  134. }
  135. func loadHashfile(fileStr string) Fdhashes {
  136. file, err := ioutil.ReadFile(fileStr)
  137. if err != nil {
  138. panic(err)
  139. }
  140. dir, _ := filepath.Split(fileStr)
  141. data := Fdhashes{Path: dir, Hashes: make(map[string]string)}
  142. err = json.Unmarshal([]byte(file), &data)
  143. if err != nil {
  144. log.Printf("can't read file %s", fileStr)
  145. }
  146. if data.Path == "" {
  147. data.Path = dir
  148. }
  149. return data
  150. }