WilliesSchematicsWorld/schematic-service-go/cmd/service.go

package main

import (
	"bufio"
	"context"
	"crypto/md5"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/robfig/cron/v3"
	api "github.com/willie68/schematic-service-go/api"
	"github.com/willie68/schematic-service-go/health"

	"github.com/willie68/schematic-service-go/internal/crypt"

	consulApi "github.com/hashicorp/consul/api"
	config "github.com/willie68/schematic-service-go/config"
	"github.com/willie68/schematic-service-go/dao"
	"github.com/willie68/schematic-service-go/logging"
	"github.com/willie68/schematic-service-go/model"

	"github.com/go-chi/chi"
	"github.com/go-chi/chi/middleware"
	"github.com/go-chi/render"

	flag "github.com/spf13/pflag"
)

/*
apiVersion implementing api version for this service
*/
const apiVersion = "1"
const servicename = "gomicro"

var port int
var sslport int
var system string
var serviceURL string
var registryURL string
var importPath string
var effectImportPath string
var apikey string
var ssl bool
var configFile string
var serviceConfig config.Config
var consulAgent *consulApi.Agent
var log logging.ServiceLogger
var c cron.Cron

func init() {
	// variables for parameter override
	ssl = false
	log.Info("init service")
	flag.IntVarP(&port, "port", "p", 0, "port of the http server.")
	flag.IntVarP(&sslport, "sslport", "t", 0, "port of the https server.")
	flag.StringVarP(&configFile, "config", "c", config.File, "this is the path and filename to the config file")
	flag.StringVarP(&serviceURL, "serviceURL", "u", "", "service url from outside")
	flag.StringVarP(&registryURL, "registryURL", "r", "", "registry url where to connect to consul")
	flag.StringVarP(&importPath, "import", "i", "", "import data from here")
	flag.StringVarP(&effectImportPath, "effect", "e", "", "effect import data from here")
}

func routes() *chi.Mux {
	//myHandler := api.NewSysAPIHandler(apikey)
	baseURL := fmt.Sprintf("/api/v%s", apiVersion)
	router := chi.NewRouter()
	router.Use(
		render.SetContentType(render.ContentTypeJSON),
		middleware.Logger,
		middleware.DefaultCompress,
		middleware.Recoverer,
		//myHandler.Handler,
		api.BasicAuth("schematic"),
	)

	router.Route("/", func(r chi.Router) {
		r.Mount(baseURL+"/config", api.ConfigRoutes())
		r.Mount(baseURL+"/files", api.FilesRoutes())
		r.Mount(baseURL+"/tags", api.TagsRoutes())
		r.Mount(baseURL+"/manufacturers", api.ManufacturersRoutes())
		r.Mount(baseURL+"/schematics", api.SchematicsRoutes())
		r.Mount(baseURL+"/users", api.UsersRoutes())
		r.Mount(baseURL+"/effects", api.EffectsRoutes())
		r.Mount(baseURL+"/effecttypes", api.EffectTypesRoutes())
		r.Mount("/health", health.Routes())
	})
	return router
}

func healthRoutes() *chi.Mux {
	router := chi.NewRouter()
	router.Use(
		render.SetContentType(render.ContentTypeJSON),
		middleware.Logger,
		middleware.DefaultCompress,
		middleware.Recoverer,
	)

	router.Route("/", func(r chi.Router) {
		r.Mount("/health", health.Routes())
	})
	return router
}

func main() {
	log.Info("starting server")
	flag.Parse()

	config.File = configFile
	if err := config.Load(); err != nil {
		log.Alertf("can't load config file: %s", err.Error())
	}
	serviceConfig = config.Get()
	initConfig()
	initGraylog()

	healthCheckConfig := health.CheckConfig(serviceConfig.HealthCheck)

	defer log.Close()

	gc := crypt.GenerateCertificate{
		Organization: "MCS Media Computer Spftware",
		Host:         "127.0.0.1",
		ValidFor:     10 * 365 * 24 * time.Hour,
		IsCA:         false,
		EcdsaCurve:   "P256",
		Ed25519Key:   true,
	}

	if serviceConfig.Sslport > 0 {
		ssl = true
		log.Info("ssl active")
	}

	storage := &dao.MongoDAO{}
	storage.InitDAO(config.Get().MongoDB)
	dao.Storage = storage

	if importPath != "" {
		go importData()
	}

	if effectImportPath != "" {
		go effectImportData()
	}

	health.InitHealthSystem(healthCheckConfig)

	apikey = getApikey()
	api.APIKey = apikey
	log.Infof("apikey: %s", apikey)
	log.Infof("ssl: %t", ssl)
	log.Infof("serviceURL: %s", serviceConfig.ServiceURL)
	if serviceConfig.RegistryURL != "" {
		log.Infof("registryURL: %s", serviceConfig.RegistryURL)
	}
	router := routes()
	walkFunc := func(method string, route string, handler http.Handler, middlewares ...func(http.Handler) http.Handler) error {
		log.Infof("%s %s", method, route)
		return nil
	}

	if err := chi.Walk(router, walkFunc); err != nil {
		log.Alertf("Logging err: %s", err.Error())
	}
	log.Info("Health routes")
	healthRouter := healthRoutes()
	if err := chi.Walk(healthRouter, walkFunc); err != nil {
		log.Alertf("Logging err: %s", err.Error())
	}

	var sslsrv *http.Server
	if ssl {
		tlsConfig, err := gc.GenerateTLSConfig()
		if err != nil {
			log.Alertf("logging err: %s", err.Error())
		}
		sslsrv = &http.Server{
			Addr:         "0.0.0.0:" + strconv.Itoa(serviceConfig.Sslport),
			WriteTimeout: time.Second * 15,
			ReadTimeout:  time.Second * 15,
			IdleTimeout:  time.Second * 60,
			Handler:      router,
			TLSConfig:    tlsConfig,
		}
		go func() {
			log.Infof("starting https server on address: %s", sslsrv.Addr)
			if err := sslsrv.ListenAndServeTLS("", ""); err != nil {
				log.Alertf("error starting server: %s", err.Error())
			}
		}()
	}

	// own http server for the healthchecks
	srv := &http.Server{
		Addr:         "0.0.0.0:" + strconv.Itoa(serviceConfig.Port),
		WriteTimeout: time.Second * 15,
		ReadTimeout:  time.Second * 15,
		IdleTimeout:  time.Second * 60,
		Handler:      healthRouter,
	}

	go func() {
		log.Infof("starting http server on address: %s", srv.Addr)
		if err := srv.ListenAndServe(); err != nil {
			log.Alertf("error starting server: %s", err.Error())
		}
	}()

	if serviceConfig.RegistryURL != "" {
		initRegistry()
	}

	// starting cron jobs
	startCron()

	osc := make(chan os.Signal, 1)
	signal.Notify(osc, os.Interrupt)
	<-osc

	log.Info("waiting for clients")
	ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
	defer cancel()

	srv.Shutdown(ctx)
	if ssl {
		sslsrv.Shutdown(ctx)
	}

	c.Stop()

	log.Info("finished")

	os.Exit(0)
}

func initGraylog() {
	log.GelfURL = serviceConfig.Logging.Gelfurl
	log.GelfPort = serviceConfig.Logging.Gelfport

	log.InitGelf()
}

func initRegistry() {
	//register to consul, if configured
	consulConfig := consulApi.DefaultConfig()
	consulURL, err := url.Parse(serviceConfig.RegistryURL)
	consulConfig.Scheme = consulURL.Scheme
	consulConfig.Address = fmt.Sprintf("%s:%s", consulURL.Hostname(), consulURL.Port())
	consulClient, err := consulApi.NewClient(consulConfig)
	if err != nil {
		log.Alertf("can't connect to consul. %v", err)
	}
	consulAgent = consulClient.Agent()

	check := new(consulApi.AgentServiceCheck)
	check.HTTP = fmt.Sprintf("%s/health/health", serviceConfig.ServiceURL)
	check.Timeout = (time.Minute * 1).String()
	check.Interval = (time.Second * 30).String()
	check.TLSSkipVerify = true
	serviceDef := &consulApi.AgentServiceRegistration{
		Name:  servicename,
		Check: check,
	}

	err = consulAgent.ServiceRegister(serviceDef)

	if err != nil {
		log.Alertf("can't register to consul. %s", err)
		time.Sleep(time.Second * 60)
	}

}

func initConfig() {
	if port > 0 {
		serviceConfig.Port = port
	}
	if sslport > 0 {
		serviceConfig.Sslport = sslport
	}
	if serviceURL != "" {
		serviceConfig.ServiceURL = serviceURL
	}
}

func getApikey() string {
	value := fmt.Sprintf("%s", servicename)
	apikey := fmt.Sprintf("%x", md5.Sum([]byte(value)))
	return strings.ToLower(apikey)
}

func startCron() {
	c = *cron.New()
	if serviceConfig.Backup.Period != "" {
		log.Info("starting cron with expression: " + serviceConfig.Backup.Period)
		c.AddFunc(serviceConfig.Backup.Period, func() { startBackup() })
	}
	c.Start()
}

func startBackup() {
	fmt.Println("Do Backup")
	if serviceConfig.Backup.Path != "" {
		dao.Storage.Backup(serviceConfig.Backup.Path)
	}
}

func importData() {
	count := 0
	dir := importPath
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if info != nil {
			if info.IsDir() {
				filepath := path + "/schematic.json"
				_, err := os.Stat(filepath)
				if !os.IsNotExist(err) {
					count++
					schematic := getSchematic(filepath)
					if schematic.Owner == "" {
						schematic.Owner = "w.klaas@gmx.de"
					}
					fileids := make(map[string]string)
					for filename, _ := range schematic.Files {
						file := path + "/" + filename
						f, err := os.Open(file)
						if err != nil {
							fmt.Printf("error: %s\n", err.Error())
						}
						defer f.Close()
						reader := bufio.NewReader(f)

						fileid, err := dao.GetStorage().AddFile(filename, reader)
						if err != nil {
							fmt.Printf("%v\n", err)
						} else {
							fmt.Printf("fileid: %s\n", fileid)
							fileids[filename] = fileid
						}
					}
					schematic.Files = fileids
					id, err := dao.GetStorage().CreateSchematic(schematic)
					if err != nil {
						fmt.Printf("%v\n", err)
					}
					fmt.Printf("%d: found %s: man: %s, model: %s\n", count, id, schematic.Manufacturer, schematic.Model)
				}
			}
		}
		return nil
	})
	if err != nil {
		fmt.Printf("%v\n", err)
	}
}

func effectImportData() {
	effectTypesImportData()
	effectModelsImportData()
}

func effectTypesImportData() {
	effectTypesFilesname := effectImportPath + "/WK_Schematic_Store$EffectType.csv"
	effectTypesReader, err := os.Open(effectTypesFilesname)
	if err != nil {
		fmt.Printf("%v\n", err)
	}
	defer effectTypesReader.Close()
	r := csv.NewReader(effectTypesReader)
	r.Comma = ';'
	r.Comment = '#'

	record, err := r.Read()
	fmt.Print(record)
	layout := "Mon, 02 Jan 2006 15:04:05 -0700" //"2006-01-02T15:04:05.000Z"

	for {
		record, err := r.Read()
		if err != nil {
			break
		}

		effectType := model.NewEffectType()
		effectType.ForeignID = record[1]
		effectType.TypeName = record[6]

		effectType.CreatedAt, err = time.Parse(layout, record[2])
		if err != nil {
			fmt.Printf("%v\n", err)
			break
		}
		effectType.LastModifiedAt, err = time.Parse(layout, record[3])
		if err != nil {
			fmt.Printf("%v\n", err)
			break
		}

		nlsString := record[4]
		nlsString = strings.TrimPrefix(nlsString, "[de->")
		nlsString = strings.TrimSuffix(nlsString, "]")
		effectType.Nls["de"] = nlsString

		typeImageString := record[5]
		typeImageString = strings.TrimPrefix(typeImageString, "images://")
		typeImageString = effectImportPath + "/" + typeImageString
		filename := filepath.Base(typeImageString)

		fileid, err := uploadFile(filename, typeImageString)
		if err != nil {
			fmt.Printf("%v\n", err)
			break
		}
		effectType.TypeImage = fileid
		dao.Storage.CreateEffectType(effectType)
	}
	if (err != nil) && (err != io.EOF) {
		fmt.Printf("%v\n", err)
	}
	fmt.Println("ready reading types")
}

func effectModelsImportData() {
	effectModelsFilesname := effectImportPath + "/WK_Schematic_Store$Effect.csv"
	effectModelsReader, err := os.Open(effectModelsFilesname)
	if err != nil {
		fmt.Printf("%v\n", err)
	}
	defer effectModelsReader.Close()
	r := csv.NewReader(effectModelsReader)
	r.Comma = ';'
	r.Comment = '#'

	record, err := r.Read()
	fmt.Print(record)
	layout := "Mon, 02 Jan 2006 15:04:05 -0700" //"2006-01-02T15:04:05.000Z"

	for {
		record, err := r.Read()
		if err != nil {
			break
		}

		effect := model.NewEffect()
		effect.ForeignID = record[1]
		effect.CreatedAt, err = time.Parse(layout, record[2])
		if err != nil {
			fmt.Printf("%v\n", err)
			break
		}
		effect.LastModifiedAt, err = time.Parse(layout, record[3])
		if err != nil {
			fmt.Printf("%v\n", err)
			break
		}
		effect.Comment = record[4]
		effect.Connector = record[5]
		effect.Current = record[6]
		effect.EffectType = record[7]

		imageString := record[8]
		imageString = strings.TrimPrefix(imageString, "images://")
		imageString = effectImportPath + "/" + imageString
		filename := filepath.Base(imageString)

		fileid, err := uploadFile(filename, imageString)
		if err != nil {
			fmt.Printf("%v\n", err)
			break
		}
		effect.Image = fileid
		effect.Model = record[9]
		effect.Manufacturer = record[10]
		effect.Voltage = record[12]

		dao.Storage.CreateEffect(effect)
	}
	if (err != nil) && (err != io.EOF) {
		fmt.Printf("%v\n", err)
	}
	fmt.Println("ready reading types")
}

func getSchematic(file string) model.Schematic {
	jsonFile, err := os.Open(file)
	// if we os.Open returns an error then handle it
	if err != nil {
		fmt.Printf("%v\n", err)
	}
	// defer the closing of our jsonFile so that we can parse it later on
	defer jsonFile.Close()
	byteValue, _ := ioutil.ReadAll(jsonFile)
	//	fmt.Println(string(byteValue))

	var schematic model.Schematic
	err = json.Unmarshal(byteValue, &schematic)
	if err != nil {
		fmt.Printf("%v\n", err)
	}
	return schematic
}

func uploadFile(filename string, file string) (string, error) {
	f, err := os.Open(file)
	if err != nil {
		return "", err
	}
	defer f.Close()
	reader := bufio.NewReader(f)

	fileid, err := dao.GetStorage().AddFile(filename, reader)
	if err != nil {
		return "", err
	}
	return fileid, nil
}