package main import ( "bufio" "context" "crypto/md5" "encoding/csv" "encoding/json" "fmt" "io" "io/ioutil" "net/http" "net/url" "os" "os/signal" "path/filepath" "strconv" "strings" "time" "github.com/robfig/cron/v3" api "github.com/willie68/schematic-service-go/api" "github.com/willie68/schematic-service-go/health" "github.com/willie68/schematic-service-go/internal/crypt" consulApi "github.com/hashicorp/consul/api" config "github.com/willie68/schematic-service-go/config" "github.com/willie68/schematic-service-go/dao" "github.com/willie68/schematic-service-go/logging" "github.com/willie68/schematic-service-go/model" "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" "github.com/go-chi/render" flag "github.com/spf13/pflag" ) /* apiVersion implementing api version for this service */ const apiVersion = "1" const servicename = "gomicro" var port int var sslport int var system string var serviceURL string var registryURL string var importPath string var effectImportPath string var apikey string var ssl bool var configFile string var serviceConfig config.Config var consulAgent *consulApi.Agent var log logging.ServiceLogger var c cron.Cron func init() { // variables for parameter override ssl = false log.Info("init service") flag.IntVarP(&port, "port", "p", 0, "port of the http server.") flag.IntVarP(&sslport, "sslport", "t", 0, "port of the https server.") flag.StringVarP(&configFile, "config", "c", config.File, "this is the path and filename to the config file") flag.StringVarP(&serviceURL, "serviceURL", "u", "", "service url from outside") flag.StringVarP(®istryURL, "registryURL", "r", "", "registry url where to connect to consul") flag.StringVarP(&importPath, "import", "i", "", "import data from here") flag.StringVarP(&effectImportPath, "effect", "e", "", "effect import data from here") } func Cors(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Access-Control-Allow-Methods", "*") w.Header().Set("Access-Control-Allow-Headers", "*") w.Header().Set("Access-Control-Allow-Credentials", "true") log.Infof("Should set headers") if r.Method == "OPTIONS" { log.Infof("Should return for OPTIONS") return } next.ServeHTTP(w, r) }) } func routes() *chi.Mux { //myHandler := api.NewSysAPIHandler(apikey) baseURL := fmt.Sprintf("/api/v%s", apiVersion) router := chi.NewRouter() router.Use( Cors, /* cors.Handler(cors.Options{ // AllowedOrigins: []string{"https://foo.com"}, // Use this to allow specific origin hosts AllowedOrigins: []string{"*"}, // AllowOriginFunc: func(r *http.Request, origin string) bool { return true }, AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "X-CSRF-Token"}, ExposedHeaders: []string{"Link"}, AllowCredentials: true, MaxAge: 300, // Maximum value not ignored by any of major browsers }), */ render.SetContentType(render.ContentTypeJSON), middleware.Logger, middleware.DefaultCompress, middleware.Recoverer, //myHandler.Handler, api.BasicAuth("schematic"), ) router.Route("/", func(r chi.Router) { r.Mount(baseURL+"/config", api.ConfigRoutes()) r.Mount(baseURL+"/files", api.FilesRoutes()) r.Mount(baseURL+"/tags", api.TagsRoutes()) r.Mount(baseURL+"/manufacturers", api.ManufacturersRoutes()) r.Mount(baseURL+"/schematics", api.SchematicsRoutes()) r.Mount(baseURL+"/users", api.UsersRoutes()) r.Mount(baseURL+"/effects", api.EffectsRoutes()) r.Mount(baseURL+"/effecttypes", api.EffectTypesRoutes()) r.Mount("/health", health.Routes()) }) return router } func healthRoutes() *chi.Mux { router := chi.NewRouter() router.Use( render.SetContentType(render.ContentTypeJSON), middleware.Logger, middleware.DefaultCompress, middleware.Recoverer, ) router.Route("/", func(r chi.Router) { r.Mount("/health", health.Routes()) }) return router } func main() { log.Info("starting server") flag.Parse() config.File = configFile if err := config.Load(); err != nil { log.Alertf("can't load config file: %s", err.Error()) } serviceConfig = config.Get() initConfig() initGraylog() healthCheckConfig := health.CheckConfig(serviceConfig.HealthCheck) defer log.Close() gc := crypt.GenerateCertificate{ Organization: "MCS Media Computer Software", Host: "127.0.0.1", ValidFor: 10 * 365 * 24 * time.Hour, IsCA: false, EcdsaCurve: "P256", Ed25519Key: true, } if serviceConfig.Sslport > 0 { ssl = true log.Info("ssl active") } storage := &dao.MongoDAO{} storage.InitDAO(config.Get().MongoDB) dao.Storage = storage if importPath != "" { go importData() } if effectImportPath != "" { go effectImportData() } health.InitHealthSystem(healthCheckConfig) apikey = getApikey() api.APIKey = apikey log.Infof("apikey: %s", apikey) log.Infof("ssl: %t", ssl) log.Infof("serviceURL: %s", serviceConfig.ServiceURL) if serviceConfig.RegistryURL != "" { log.Infof("registryURL: %s", serviceConfig.RegistryURL) } router := routes() walkFunc := func(method string, route string, handler http.Handler, middlewares ...func(http.Handler) http.Handler) error { log.Infof("%s %s", method, route) return nil } if err := chi.Walk(router, walkFunc); err != nil { log.Alertf("Logging err: %s", err.Error()) } log.Info("Health routes") healthRouter := healthRoutes() if err := chi.Walk(healthRouter, walkFunc); err != nil { log.Alertf("Logging err: %s", err.Error()) } var sslsrv *http.Server var srv *http.Server if ssl { tlsConfig, err := gc.GenerateTLSConfig() if err != nil { log.Alertf("logging err: %s", err.Error()) } sslsrv = &http.Server{ Addr: "0.0.0.0:" + strconv.Itoa(serviceConfig.Sslport), WriteTimeout: time.Second * 15, ReadTimeout: time.Second * 15, IdleTimeout: time.Second * 60, Handler: router, TLSConfig: tlsConfig, } go func() { log.Infof("starting https server on address: %s", sslsrv.Addr) if err := sslsrv.ListenAndServeTLS("", ""); err != nil { log.Alertf("error starting server: %s", err.Error()) } }() srv = &http.Server{ Addr: "0.0.0.0:" + strconv.Itoa(serviceConfig.Port), WriteTimeout: time.Second * 15, ReadTimeout: time.Second * 15, IdleTimeout: time.Second * 60, Handler: healthRouter, } go func() { log.Infof("starting http server on address: %s", srv.Addr) if err := srv.ListenAndServe(); err != nil { log.Alertf("error starting server: %s", err.Error()) } }() } else { // own http server for the healthchecks srv = &http.Server{ Addr: "0.0.0.0:" + strconv.Itoa(serviceConfig.Port), WriteTimeout: time.Second * 15, ReadTimeout: time.Second * 15, IdleTimeout: time.Second * 60, Handler: router, } go func() { log.Infof("starting http server on address: %s", srv.Addr) if err := srv.ListenAndServe(); err != nil { log.Alertf("error starting server: %s", err.Error()) } }() } if serviceConfig.RegistryURL != "" { initRegistry() } // starting cron jobs startCron() osc := make(chan os.Signal, 1) signal.Notify(osc, os.Interrupt) <-osc log.Info("waiting for clients") ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer cancel() srv.Shutdown(ctx) if ssl { sslsrv.Shutdown(ctx) } c.Stop() log.Info("finished") os.Exit(0) } func initGraylog() { log.GelfURL = serviceConfig.Logging.Gelfurl log.GelfPort = serviceConfig.Logging.Gelfport log.InitGelf() } func initRegistry() { //register to consul, if configured consulConfig := consulApi.DefaultConfig() consulURL, err := url.Parse(serviceConfig.RegistryURL) consulConfig.Scheme = consulURL.Scheme consulConfig.Address = fmt.Sprintf("%s:%s", consulURL.Hostname(), consulURL.Port()) consulClient, err := consulApi.NewClient(consulConfig) if err != nil { log.Alertf("can't connect to consul. %v", err) } consulAgent = consulClient.Agent() check := new(consulApi.AgentServiceCheck) check.HTTP = fmt.Sprintf("%s/health/health", serviceConfig.ServiceURL) check.Timeout = (time.Minute * 1).String() check.Interval = (time.Second * 30).String() check.TLSSkipVerify = true serviceDef := &consulApi.AgentServiceRegistration{ Name: servicename, Check: check, } err = consulAgent.ServiceRegister(serviceDef) if err != nil { log.Alertf("can't register to consul. %s", err) time.Sleep(time.Second * 60) } } func initConfig() { if port > 0 { serviceConfig.Port = port } if sslport > 0 { serviceConfig.Sslport = sslport } if serviceURL != "" { serviceConfig.ServiceURL = serviceURL } } func getApikey() string { value := fmt.Sprintf("%s", servicename) apikey := fmt.Sprintf("%x", md5.Sum([]byte(value))) return strings.ToLower(apikey) } func startCron() { c = *cron.New() if serviceConfig.Backup.Period != "" { log.Info("starting cron with expression: " + serviceConfig.Backup.Period) c.AddFunc(serviceConfig.Backup.Period, func() { startBackup() }) } c.Start() } func startBackup() { fmt.Println("Do Backup") if serviceConfig.Backup.Path != "" { dao.Storage.Backup(serviceConfig.Backup.Path) } } func importData() { count := 0 dir := importPath err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if info != nil { if info.IsDir() { filepath := path + "/schematic.json" _, err := os.Stat(filepath) if !os.IsNotExist(err) { count++ schematic := getSchematic(filepath) if schematic.Owner == "" { schematic.Owner = "w.klaas@gmx.de" } fileids := make(map[string]string) for filename, _ := range schematic.Files { file := path + "/" + filename f, err := os.Open(file) if err != nil { fmt.Printf("error: %s\n", err.Error()) } defer f.Close() reader := bufio.NewReader(f) fileid, err := dao.GetStorage().AddFile(filename, reader) if err != nil { fmt.Printf("%v\n", err) } else { fmt.Printf("fileid: %s\n", fileid) fileids[filename] = fileid } } schematic.Files = fileids id, err := dao.GetStorage().CreateSchematic(schematic) if err != nil { fmt.Printf("%v\n", err) } fmt.Printf("%d: found %s: man: %s, model: %s\n", count, id, schematic.Manufacturer, schematic.Model) } } } return nil }) if err != nil { fmt.Printf("%v\n", err) } } func effectImportData() { effectTypesImportData() effectModelsImportData() } func effectTypesImportData() { effectTypesFilesname := effectImportPath + "/WK_Schematic_Store$EffectType.csv" effectTypesReader, err := os.Open(effectTypesFilesname) if err != nil { fmt.Printf("%v\n", err) } defer effectTypesReader.Close() r := csv.NewReader(effectTypesReader) r.Comma = ';' r.Comment = '#' record, err := r.Read() fmt.Print(record) layout := "Mon, 02 Jan 2006 15:04:05 -0700" //"2006-01-02T15:04:05.000Z" for { record, err := r.Read() if err != nil { break } effectType := model.NewEffectType() effectType.ForeignID = record[1] effectType.TypeName = record[6] effectType.CreatedAt, err = time.Parse(layout, record[2]) if err != nil { fmt.Printf("%v\n", err) break } effectType.LastModifiedAt, err = time.Parse(layout, record[3]) if err != nil { fmt.Printf("%v\n", err) break } nlsString := record[4] nlsString = strings.TrimPrefix(nlsString, "[de->") nlsString = strings.TrimSuffix(nlsString, "]") effectType.Nls["de"] = nlsString typeImageString := record[5] typeImageString = strings.TrimPrefix(typeImageString, "images://") typeImageString = effectImportPath + "/" + typeImageString filename := filepath.Base(typeImageString) fileid, err := uploadFile(filename, typeImageString) if err != nil { fmt.Printf("%v\n", err) break } effectType.TypeImage = fileid dao.Storage.CreateEffectType(effectType) } if (err != nil) && (err != io.EOF) { fmt.Printf("%v\n", err) } fmt.Println("ready reading types") } func effectModelsImportData() { effectModelsFilesname := effectImportPath + "/WK_Schematic_Store$Effect.csv" effectModelsReader, err := os.Open(effectModelsFilesname) if err != nil { fmt.Printf("%v\n", err) } defer effectModelsReader.Close() r := csv.NewReader(effectModelsReader) r.Comma = ';' r.Comment = '#' record, err := r.Read() fmt.Print(record) layout := "Mon, 02 Jan 2006 15:04:05 -0700" //"2006-01-02T15:04:05.000Z" for { record, err := r.Read() if err != nil { break } effect := model.NewEffect() effect.ForeignID = record[1] effect.CreatedAt, err = time.Parse(layout, record[2]) if err != nil { fmt.Printf("%v\n", err) break } effect.LastModifiedAt, err = time.Parse(layout, record[3]) if err != nil { fmt.Printf("%v\n", err) break } effect.Comment = record[4] effect.Connector = record[5] effect.Current = record[6] effect.EffectType = record[7] imageString := record[8] imageString = strings.TrimPrefix(imageString, "images://") imageString = effectImportPath + "/" + imageString filename := filepath.Base(imageString) fileid, err := uploadFile(filename, imageString) if err != nil { fmt.Printf("%v\n", err) break } effect.Image = fileid effect.Model = record[9] effect.Manufacturer = record[10] effect.Voltage = record[12] dao.Storage.CreateEffect(effect) } if (err != nil) && (err != io.EOF) { fmt.Printf("%v\n", err) } fmt.Println("ready reading types") } func getSchematic(file string) model.Schematic { jsonFile, err := os.Open(file) // if we os.Open returns an error then handle it if err != nil { fmt.Printf("%v\n", err) } // defer the closing of our jsonFile so that we can parse it later on defer jsonFile.Close() byteValue, _ := ioutil.ReadAll(jsonFile) // fmt.Println(string(byteValue)) var schematic model.Schematic err = json.Unmarshal(byteValue, &schematic) if err != nil { fmt.Printf("%v\n", err) } return schematic } func uploadFile(filename string, file string) (string, error) { f, err := os.Open(file) if err != nil { return "", err } defer f.Close() reader := bufio.NewReader(f) fileid, err := dao.GetStorage().AddFile(filename, reader) if err != nil { return "", err } return fileid, nil }