bisschen aufgeräumt, cron job hinzugefügt, backup implementiert

schematic-service-go/api/endpoints.go

@@ -1,12 +1,13 @@
 package api
 
 import (
-	"log"
 	"net/http"
 	"time"
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
+	"github.com/google/martian/log"
+	"github.com/willie68/schematic-service-go/config"
 	"github.com/willie68/schematic-service-go/dao"
 )
 
@@ -34,11 +35,9 @@ ConfigRoutes getting all routes for the config endpoint
 */
 func ConfigRoutes() *chi.Mux {
 	router := chi.NewRouter()
-	router.Post("/", PostConfigEndpoint)
 	router.Get("/", GetConfigEndpoint)
-	router.Delete("/", DeleteConfigEndpoint)
-	router.Get("/size", GetConfigSizeEndpoint)
 	router.Delete("/dropall", DropAllEndpoint)
+	router.Post("/backup", PostBackupEndpoint)
 	return router
 }
 
@@ -47,64 +46,21 @@ GetConfigEndpoint getting if a store for a tenant is initialised
 because of the automatic store creation, the value is more likely that data is stored for this tenant
 */
 func GetConfigEndpoint(response http.ResponseWriter, req *http.Request) {
-	tenant := getTenant(req)
-	if tenant == "" {
-		Msg(response, http.StatusBadRequest, "tenant not set")
-		return
-	}
-	c := ConfigDescription{
-		StoreID:  "myNewStore",
-		TenantID: tenant,
-		Size:     1234567,
-	}
-	render.JSON(response, req, c)
+	Msg(response, http.StatusNotImplemented, "not im plemented yet")
 }
 
 /*
 PostConfigEndpoint create a new store for a tenant
 because of the automatic store creation, this method will always return 201
 */
-func PostConfigEndpoint(response http.ResponseWriter, req *http.Request) {
-	tenant := getTenant(req)
-	if tenant == "" {
-		Msg(response, http.StatusBadRequest, "tenant not set")
-		return
-	}
-	log.Printf("create store for tenant %s", tenant)
-	render.Status(req, http.StatusCreated)
-	render.JSON(response, req, tenant)
-}
-
-/*
-DeleteConfigEndpoint deleting store for a tenant, this will automatically delete all data in the store
-*/
-func DeleteConfigEndpoint(response http.ResponseWriter, req *http.Request) {
-	tenant := getTenant(req)
-	if tenant == "" {
-		Msg(response, http.StatusBadRequest, "tenant not set")
-		return
-	}
-	render.JSON(response, req, tenant)
-}
-
-/*
-GetConfigSizeEndpoint size of the store for a tenant
-*/
-func GetConfigSizeEndpoint(response http.ResponseWriter, req *http.Request) {
-	tenant := getTenant(req)
-	if tenant == "" {
-		Msg(response, http.StatusBadRequest, "tenant not set")
-		return
-	}
-
-	render.JSON(response, req, tenant)
-}
-
-/*
-getTenant getting the tenant from the request
-*/
-func getTenant(req *http.Request) string {
-	return req.Header.Get(TenantHeader)
+func PostBackupEndpoint(response http.ResponseWriter, req *http.Request) {
+	go func() {
+		err := dao.GetStorage().Backup(config.Get().Backup.Path)
+		if err != nil {
+			log.Infof("error in backup: %v", err)
+		}
+	}()
+	render.JSON(response, req, "backup started.")
 }
 
 //DropAllEndpoint dropping all data

schematic-service-go/cmd/service.go

@@ -18,6 +18,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/robfig/cron/v3"
 	api "github.com/willie68/schematic-service-go/api"
 	"github.com/willie68/schematic-service-go/health"
 
@@ -55,6 +56,7 @@ var configFile string
 var serviceConfig config.Config
 var consulAgent *consulApi.Agent
 var log logging.ServiceLogger
+var c cron.Cron
 
 func init() {
 	// variables for parameter override
@@ -143,7 +145,6 @@ func main() {
 
 	storage := &dao.MongoDAO{}
 	storage.InitDAO(config.Get().MongoDB)
-	//storage := &dao.SimpleDAO{Path: "E:/temp/schematics/"}
 	dao.Storage = storage
 
 	if importPath != "" {
@@ -221,9 +222,12 @@ func main() {
 		initRegistry()
 	}
 
-	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt)
-	<-c
+	// starting cron jobs
+	startCron()
+
+	osc := make(chan os.Signal, 1)
+	signal.Notify(osc, os.Interrupt)
+	<-osc
 
 	log.Info("waiting for clients")
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
@@ -234,6 +238,8 @@ func main() {
 		sslsrv.Shutdown(ctx)
 	}
 
+	c.Stop()
+
 	log.Info("finished")
 
 	os.Exit(0)
@@ -295,6 +301,22 @@ func getApikey() string {
 	return strings.ToLower(apikey)
 }
 
+func startCron() {
+	c = *cron.New()
+	if serviceConfig.Backup.Period != "" {
+		log.Info("starting cron with expression: " + serviceConfig.Backup.Period)
+		c.AddFunc(serviceConfig.Backup.Period, func() { startBackup() })
+	}
+	c.Start()
+}
+
+func startBackup() {
+	fmt.Println("Do Backup")
+	if serviceConfig.Backup.Path != "" {
+		dao.Storage.Backup(serviceConfig.Backup.Path)
+	}
+}
+
 func importData() {
 	count := 0
 	dir := importPath

schematic-service-go/config/config.go

@@ -14,11 +14,18 @@ type Config struct {
 	SecretFile string  `yaml:"secretfile"`
 	Logging    Logging `yaml:"logging"`
 
+	Backup Backup `yaml:"backup"`
+
 	HealthCheck HealthCheck `yaml:"healthcheck"`
 
 	MongoDB MongoDB `yaml: "mongodb"`
 }
 
+type Backup struct {
+	Path   string `yaml:"path"`
+	Period string `yaml:"period"`
+}
+
 type Logging struct {
 	Gelfurl  string `yaml:"gelf-url"`
 	Gelfport int    `yaml:"gelf-port"`

schematic-service-go/configs/service.yaml

@@ -10,6 +10,10 @@ registryURL: http://127.0.0.1:8500
 systemID: easy1
 #sercret file for storing usernames and passwords
 secretfile: /tmp/storage/config/secret.yaml
+backup:
+    path: e:/temp/backup/schematics/new
+    #\\OMV/backup/schematic/new
+    period: "@every 24h"
 
 logging:
     gelf-url: 127.0.0.1
@@ -17,3 +21,12 @@ logging:
 
 healthcheck:
     period: 30
+
+mongodb:
+    host: 127.0.0.1
+    port: 27017
+    username:
+    password:
+    authdb: schematics
+    database: schematics
+    

schematic-service-go/configs/serviceLocal.yaml

@@ -10,6 +10,10 @@ registryURL:
 systemID: easy1
 #sercret file for storing usernames and passwords
 secretfile: configs/secret.yaml
+backup:
+    path: e:/temp/backup/schematics/new
+    #\\OMV/backup/schematic/new
+    period: "@every 24h"
 
 logging:
     gelf-url:

schematic-service-go/dao/mongodao.go

@@ -6,7 +6,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
+	"os"
 	"sort"
 	"strings"
 	"time"
@@ -610,12 +612,11 @@ func (m *MongoDAO) GetFile(fileid string, stream io.Writer) error {
 		log.Print(err)
 		return err
 	}
-	dStream, err := m.bucket.DownloadToStream(objectID, stream)
+	_, err = m.bucket.DownloadToStream(objectID, stream)
 	if err != nil {
 		log.Print(err)
 		return err
 	}
-	fmt.Printf("File size to download: %v \n", dStream)
 	return nil
 }
 
@@ -1124,6 +1125,320 @@ func (m *MongoDAO) ChangePWD(username string, newpassword string, oldpassword st
 	return nil
 }
 
+//Backup pinging the mongoDao
+func (m *MongoDAO) Backup(path string) error {
+	// writung users
+	err := m.backupUsers(path)
+	if err != nil {
+		return err
+	}
+	// writing tags
+	err = m.backupTags(path)
+	if err != nil {
+		return err
+	}
+	// writing manufacturers
+	err = m.backupManufacturers(path)
+	if err != nil {
+		return err
+	}
+	// writing schematics
+	err = m.backupSchematics(path)
+	if err != nil {
+		return err
+	}
+	// writing effect categories
+	err = m.backupEffectTypes(path)
+	if err != nil {
+		return err
+	}
+	// writing effects
+	err = m.backupEffects(path)
+	if err != nil {
+		return err
+	}
+	return err
+}
+
+func (m *MongoDAO) backupUsers(path string) error {
+	path = path + "/users"
+	os.MkdirAll(path, os.ModePerm)
+
+	ctx, _ := context.WithTimeout(context.Background(), timeout)
+	collection := m.database.Collection(usersCollectionName)
+	cursor, err := collection.Find(ctx, bson.M{})
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer cursor.Close(ctx)
+	count := 0
+	fmt.Print("backup users: ")
+	for cursor.Next(ctx) {
+		var user model.User
+		if err = cursor.Decode(&user); err != nil {
+			log.Fatal(err)
+			return err
+		} else {
+			user.Name = strings.ToLower(user.Name)
+			user.Password = BuildPasswordHash(user.Password)
+			data, err := json.Marshal(user)
+			if err != nil {
+				return err
+			}
+			filename := path + "/" + user.ID.Hex() + ".json"
+			ioutil.WriteFile(filename, data, os.ModePerm)
+			count++
+			if count%100 == 0 {
+				fmt.Print(".")
+			}
+		}
+	}
+	fmt.Println()
+	return nil
+}
+
+func (m *MongoDAO) backupTags(path string) error {
+	path = path + "/tags"
+	os.MkdirAll(path, os.ModePerm)
+
+	ctx, _ := context.WithTimeout(context.Background(), timeout)
+	collection := m.database.Collection(tagsCollectionName)
+	cursor, err := collection.Find(ctx, bson.M{})
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer cursor.Close(ctx)
+	count := 0
+	fmt.Print("backup tags: ")
+	for cursor.Next(ctx) {
+		var model model.Tag
+		if err = cursor.Decode(&model); err != nil {
+			log.Fatal(err)
+			return err
+		} else {
+			data, err := json.Marshal(model)
+			if err != nil {
+				return err
+			}
+			filename := path + "/" + model.ID.Hex() + ".json"
+			ioutil.WriteFile(filename, data, os.ModePerm)
+			count++
+			if count%100 == 0 {
+				fmt.Print(".")
+			}
+		}
+	}
+	fmt.Println()
+	return nil
+}
+
+func (m *MongoDAO) backupManufacturers(path string) error {
+	path = path + "/manufacturers"
+	os.MkdirAll(path, os.ModePerm)
+
+	ctx, _ := context.WithTimeout(context.Background(), timeout)
+	collection := m.database.Collection(manufacturersCollectionName)
+	cursor, err := collection.Find(ctx, bson.M{})
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer cursor.Close(ctx)
+	count := 0
+	fmt.Print("backup manufacturers: ")
+	for cursor.Next(ctx) {
+		var model model.Manufacturer
+		if err = cursor.Decode(&model); err != nil {
+			log.Fatal(err)
+			return err
+		} else {
+			data, err := json.Marshal(model)
+			if err != nil {
+				return err
+			}
+			filename := path + "/" + model.ID.Hex() + ".json"
+			ioutil.WriteFile(filename, data, os.ModePerm)
+			count++
+			if count%100 == 0 {
+				fmt.Print(".")
+			}
+		}
+	}
+	fmt.Println()
+	return nil
+}
+func (m *MongoDAO) backupSchematics(path string) error {
+	path = path + "/schematics"
+	os.MkdirAll(path, os.ModePerm)
+
+	ctx, _ := context.WithTimeout(context.Background(), timeout)
+	collection := m.database.Collection(schematicsCollectionName)
+	cursor, err := collection.Find(ctx, bson.M{})
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer cursor.Close(ctx)
+	count := 0
+	fmt.Print("backup schematics: ")
+	for cursor.Next(ctx) {
+		var model model.Schematic
+		if err = cursor.Decode(&model); err != nil {
+			log.Fatal(err)
+			return err
+		} else {
+			folder := path + "/" + model.ID.Hex()
+			os.MkdirAll(folder, os.ModePerm)
+
+			for _, fileid := range model.Files {
+				if fileid != "" {
+
+					filename, err := m.GetFilename(fileid)
+					if err != nil {
+						return err
+					}
+
+					imagename := folder + "/" + filename
+					file, err := os.Create(imagename)
+					if err != nil {
+						return err
+					}
+					defer file.Close()
+					err = m.GetFile(fileid, file)
+					if err != nil {
+						return err
+					}
+				}
+			}
+			data, err := json.Marshal(model)
+			if err != nil {
+				return err
+			}
+
+			filename := folder + "/schematic.json"
+			ioutil.WriteFile(filename, data, os.ModePerm)
+			count++
+			if count%100 == 0 {
+				fmt.Print(".")
+			}
+		}
+	}
+	fmt.Println()
+	return nil
+}
+func (m *MongoDAO) backupEffectTypes(path string) error {
+	path = path + "/effecttypes"
+	os.MkdirAll(path, os.ModePerm)
+
+	ctx, _ := context.WithTimeout(context.Background(), timeout)
+	collection := m.database.Collection(effectTypesCollectionName)
+	cursor, err := collection.Find(ctx, bson.M{})
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer cursor.Close(ctx)
+	count := 0
+	fmt.Print("backup effecttypes: ")
+	for cursor.Next(ctx) {
+		var model model.EffectType
+		if err = cursor.Decode(&model); err != nil {
+			log.Fatal(err)
+			return err
+		} else {
+			folder := path + "/" + model.ID.Hex()
+			os.MkdirAll(folder, os.ModePerm)
+
+			fileid := model.TypeImage
+			if fileid != "" {
+
+				filename, err := m.GetFilename(fileid)
+				if err != nil {
+					return err
+				}
+
+				imagename := folder + "/" + filename
+				file, err := os.Create(imagename)
+				if err != nil {
+					return err
+				}
+				defer file.Close()
+				err = m.GetFile(fileid, file)
+				if err != nil {
+					return err
+				}
+				model.TypeImage = filename
+			}
+			data, err := json.Marshal(model)
+			if err != nil {
+				return err
+			}
+
+			filename := folder + "/effecttype.json"
+			ioutil.WriteFile(filename, data, os.ModePerm)
+			count++
+			if count%100 == 0 {
+				fmt.Print(".")
+			}
+		}
+	}
+	fmt.Println()
+	return nil
+}
+func (m *MongoDAO) backupEffects(path string) error {
+	path = path + "/effects"
+	os.MkdirAll(path, os.ModePerm)
+
+	ctx, _ := context.WithTimeout(context.Background(), timeout)
+	collection := m.database.Collection(effectsCollectionName)
+	cursor, err := collection.Find(ctx, bson.M{})
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer cursor.Close(ctx)
+	count := 0
+	fmt.Print("backup effects: ")
+	for cursor.Next(ctx) {
+		var model model.Effect
+		if err = cursor.Decode(&model); err != nil {
+			log.Fatal(err)
+			return err
+		} else {
+			folder := path + "/" + model.ID.Hex()
+			os.MkdirAll(folder, os.ModePerm)
+			fileid := model.Image
+			if fileid != "" {
+				filename, err := m.GetFilename(fileid)
+				if err != nil {
+					return err
+				}
+
+				imagename := folder + "/" + filename
+				file, err := os.Create(imagename)
+				if err != nil {
+					return err
+				}
+				defer file.Close()
+				err = m.GetFile(fileid, file)
+				if err != nil {
+					return err
+				}
+				model.Image = filename
+			}
+			data, err := json.Marshal(model)
+			if err != nil {
+				return err
+			}
+
+			filename := folder + "/effect.json"
+			ioutil.WriteFile(filename, data, os.ModePerm)
+			count++
+			if count%100 == 0 {
+				fmt.Print(".")
+			}
+		}
+	}
+	fmt.Println()
+	return nil
+}
+
 // Ping pinging the mongoDao
 func (m *MongoDAO) Ping() error {
 	if !m.initialised {

schematic-service-go/dao/storageDao.go

@@ -47,6 +47,7 @@ type StorageDao interface {
 	DropAll()
 	Ping() error
 
+	Backup(path string) error
 	Stop()
 }
 

schematic-service-go/go.mod

@@ -8,10 +8,13 @@ require (
 	github.com/go-chi/chi v4.0.3+incompatible
 	github.com/go-chi/render v1.0.1
 	github.com/go-delve/delve v1.4.0 // indirect
+	github.com/google/martian v2.1.0+incompatible
 	github.com/google/uuid v1.1.1
 	github.com/hashicorp/consul v1.7.1 // indirect
 	github.com/hashicorp/consul/api v1.4.0
 	github.com/prometheus/client_golang v1.4.1 // indirect
+	github.com/robfig/cron v1.2.0 // indirect
+	github.com/robfig/cron/v3 v3.0.0
 	github.com/spf13/pflag v1.0.5
 	go.mongodb.org/mongo-driver v1.3.1
 	golang.org/x/net v0.0.0-20200202094626-16171245cfb2 // indirect

schematic-service-go/go.sum

@@ -108,6 +108,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
@@ -283,6 +285,10 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03/go.mod h1:gRAiPF5C5Nd0eyyRdqIu9qTiFSoZzpTq727b5B8fkkU=
+github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
+github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
+github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E=
+github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=