GoHash.go 4.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199
  1. package main
  2. import (
  3. "crypto/sha256"
  4. "encoding/hex"
  5. "encoding/json"
  6. "fmt"
  7. "io"
  8. "io/ioutil"
  9. "log"
  10. "os"
  11. "path/filepath"
  12. "runtime"
  13. "sync"
  14. "time"
  15. flag "github.com/spf13/pflag"
  16. )
  17. // Fdhashes struct for holding all informations about one folder.
  18. type Fdhashes struct {
  19. Path string
  20. Hashes map[string]string
  21. Times map[string]time.Time
  22. }
  23. var hashes map[string]Fdhashes
  24. var wg sync.WaitGroup
  25. var mu sync.RWMutex
  26. var driveLetter string
  27. var rewrite bool
  28. func init() {
  29. flag.BoolVarP(&rewrite, "rewrite", "r", false, "rewrite all fhhashes files.")
  30. }
  31. func main() {
  32. log.Println("starting GoHash")
  33. runtime.GOMAXPROCS(5)
  34. hashes = make(map[string]Fdhashes)
  35. flag.Parse()
  36. myFile := flag.Arg(0)
  37. file, err := os.Stat(myFile)
  38. if os.IsNotExist(err) {
  39. log.Fatalln("File does not exists:", myFile)
  40. }
  41. if file.IsDir() {
  42. log.Println("start with folder:", myFile)
  43. driveLetter = ""
  44. if runtime.GOOS == "windows" {
  45. driveLetter = filepath.VolumeName(myFile) + "/"
  46. }
  47. processFolder(myFile)
  48. } else {
  49. log.Printf("file %s has hash %s\n", myFile, getSha256Hash(myFile))
  50. }
  51. fmt.Println("waiting")
  52. wg.Wait()
  53. log.Println("done")
  54. }
  55. func getSha256Hash(fileStr string) string {
  56. f, err := os.Open(fileStr)
  57. if err != nil {
  58. log.Fatal(err)
  59. }
  60. defer f.Close()
  61. h := sha256.New()
  62. if _, err := io.Copy(h, f); err != nil {
  63. log.Fatal(err)
  64. }
  65. return hex.EncodeToString(h.Sum(nil))
  66. }
  67. var lock1 = sync.RWMutex{}
  68. var lock2 = sync.RWMutex{}
  69. func outputHash(fileStr string) {
  70. var hashFile Fdhashes
  71. doHash := true
  72. defer wg.Done()
  73. dir, fileName := filepath.Split(fileStr)
  74. if fileName == ".fdhashes3" {
  75. return
  76. }
  77. // checking if hash is present
  78. mu.Lock()
  79. hashFile, ok := hashes[dir]
  80. if !ok {
  81. _, err := os.Stat(dir + ".fdhashes3")
  82. if os.IsNotExist(err) {
  83. hashFile = Fdhashes{Path: dir, Hashes: make(map[string]string), Times: make(map[string]time.Time)}
  84. } else {
  85. hashFile = loadHashfile(dir + ".fdhashes3")
  86. }
  87. hashes[dir] = hashFile
  88. saveHashfile(hashFile)
  89. }
  90. lock1.RLock()
  91. _, ok = hashFile.Hashes[fileName]
  92. lock1.RUnlock()
  93. mu.Unlock()
  94. doHash = !ok
  95. // checking if dattime is identically
  96. file, _ := os.Stat(fileStr)
  97. time := file.ModTime()
  98. lock2.RLock()
  99. savedTime, ok := hashFile.Times[fileName]
  100. lock2.RUnlock()
  101. if !time.Equal(savedTime) || !ok {
  102. doHash = true
  103. }
  104. if doHash {
  105. hash := getSha256Hash(fileStr)
  106. mu.Lock()
  107. lock1.Lock()
  108. hashFile.Hashes[fileName] = hash
  109. lock1.Unlock()
  110. lock2.Lock()
  111. hashFile.Times[fileName] = time
  112. lock2.Unlock()
  113. hashes[dir] = hashFile
  114. saveHashfile(hashFile)
  115. mu.Unlock()
  116. log.Printf("file \"%s\" has hash \"%s\"\n", fileStr, hash)
  117. }
  118. }
  119. var count int
  120. var addWork int
  121. func processFolder(folder string) {
  122. count = 0
  123. addWork = 0
  124. err := filepath.Walk(folder, func(path string, info os.FileInfo, err error) error {
  125. count++
  126. if (count % 100) == 0 {
  127. fmt.Print(".")
  128. }
  129. if (count % 10000) == 0 {
  130. fmt.Println()
  131. }
  132. filename := info.Name()
  133. if filename[0:1] != "." {
  134. if info.IsDir() {
  135. fmt.Println(path)
  136. }
  137. if !info.IsDir() {
  138. addWork++
  139. wg.Add(1)
  140. go outputHash(path)
  141. if addWork > 100 {
  142. fmt.Println("x")
  143. wg.Wait()
  144. addWork = 0
  145. }
  146. }
  147. }
  148. return nil
  149. })
  150. if err != nil {
  151. panic(err)
  152. }
  153. }
  154. func saveHashfile(hashFile Fdhashes) {
  155. b, err := json.Marshal(hashFile)
  156. if err != nil {
  157. fmt.Println(err)
  158. return
  159. }
  160. err = ioutil.WriteFile(hashFile.Path+".fdhashes3", b, 0644)
  161. if err != nil {
  162. panic(err)
  163. }
  164. }
  165. func loadHashfile(fileStr string) Fdhashes {
  166. dir, _ := filepath.Split(fileStr)
  167. data := Fdhashes{Path: dir, Hashes: make(map[string]string), Times: make(map[string]time.Time)}
  168. if !rewrite {
  169. file, err := ioutil.ReadFile(fileStr)
  170. if err != nil {
  171. panic(err)
  172. }
  173. err = json.Unmarshal([]byte(file), &data)
  174. if err != nil {
  175. log.Printf("can't read file %s", fileStr)
  176. }
  177. }
  178. if data.Path == "" {
  179. data.Path = dir
  180. }
  181. return data
  182. }