more performance

.vscode/launch.json

@@ -4,7 +4,6 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
-
         {
             "name": "GoHash",
             "type": "go",
@@ -12,7 +11,7 @@
             "mode": "auto",
             "program": "${fileDirname}",
             "env": {},
-            "args": ["F:\\noShare\\pics"]
+            "args": ["F:\\noShare\\pics\\tumblr\\data"]
         }
     ]
 }

GoHash.go

@@ -22,6 +22,7 @@ type Fdhashes struct {
 	Path   string
 	Hashes map[string]string
 	Times  map[string]time.Time
+	Dirty  bool
 }
 
 var hashes map[string]Fdhashes
@@ -59,6 +60,7 @@ func main() {
 
 	fmt.Println("waiting")
 	wg.Wait()
+	saveAllHashFiles()
 	log.Println("done")
 }
 
@@ -93,12 +95,11 @@ func outputHash(fileStr string) {
 	if !ok {
 		_, err := os.Stat(dir + ".fdhashes3")
 		if os.IsNotExist(err) {
-			hashFile = Fdhashes{Path: dir, Hashes: make(map[string]string), Times: make(map[string]time.Time)}
+			hashFile = Fdhashes{Path: dir, Hashes: make(map[string]string), Times: make(map[string]time.Time), Dirty: true}
 		} else {
 			hashFile = loadHashfile(dir + ".fdhashes3")
 		}
 		hashes[dir] = hashFile
-		saveHashfile(hashFile)
 	}
 	lock1.RLock()
 	_, ok = hashFile.Hashes[fileName]
@@ -120,11 +121,12 @@ func outputHash(fileStr string) {
 		lock1.Lock()
 		hashFile.Hashes[fileName] = hash
 		lock1.Unlock()
+
 		lock2.Lock()
 		hashFile.Times[fileName] = time
 		lock2.Unlock()
+		saveHashfile(&hashFile)
 		hashes[dir] = hashFile
-		saveHashfile(hashFile)
 		mu.Unlock()
 		log.Printf("file \"%s\" has hash \"%s\"\n", fileStr, hash)
 	}
@@ -153,9 +155,10 @@ func processFolder(folder string) {
 				addWork++
 				wg.Add(1)
 				go outputHash(path)
-				if addWork > 100 {
+				if addWork > 1000 {
 					fmt.Println("x")
 					wg.Wait()
+					saveAllHashFiles()
 					addWork = 0
 				}
 			}
@@ -167,21 +170,39 @@ func processFolder(folder string) {
 	}
 }
 
-func saveHashfile(hashFile Fdhashes) {
-	b, err := json.Marshal(hashFile)
-	if err != nil {
-		fmt.Println(err)
-		return
+func saveHashfile(hashFile *Fdhashes) {
+	hashFile.Dirty = true
+}
+
+func saveAllHashFiles() {
+	hashList := make([]Fdhashes, 0)
+
+	for _, hashFile := range hashes {
+		if hashFile.Dirty {
+			hashFile.Dirty = false
+			b, err := json.Marshal(hashFile)
+			if err != nil {
+				fmt.Println(err)
+				return
+			}
+			err = ioutil.WriteFile(hashFile.Path+".fdhashes3", b, 0644)
+			if err != nil {
+				panic(err)
+			}
+			hashList = append(hashList, hashFile)
+		}
 	}
-	err = ioutil.WriteFile(hashFile.Path+".fdhashes3", b, 0644)
-	if err != nil {
-		panic(err)
+
+	hashes = make(map[string]Fdhashes)
+	for _, hashFile := range hashList {
+		hashes[hashFile.Path] = hashFile
 	}
+
 }
 
 func loadHashfile(fileStr string) Fdhashes {
 	dir, _ := filepath.Split(fileStr)
-	data := Fdhashes{Path: dir, Hashes: make(map[string]string), Times: make(map[string]time.Time)}
+	data := Fdhashes{Path: dir, Hashes: make(map[string]string), Times: make(map[string]time.Time), Dirty: false}
 	if !rewrite {
 		file, err := ioutil.ReadFile(fileStr)
 		if err != nil {
@@ -192,8 +213,9 @@ func loadHashfile(fileStr string) Fdhashes {
 			log.Printf("can't read file %s", fileStr)
 		}
 	}
-	if data.Path == "" {
+	if data.Path != dir {
 		data.Path = dir
+		data.Dirty = true
 	}
 	return data
 }