changing source control reference

MessageService/Dockerfile

@@ -0,0 +1,90 @@
+##### BUILDER #####
+
+FROM golang:1.14-alpine3.11 as builder
+
+## Task: Install build deps
+
+# hadolint ignore=DL3018
+RUN set -eux; \
+    apk add --no-progress --quiet --no-cache --upgrade --virtual .build-deps \
+        gcc \
+        git \
+        musl-dev
+
+## Task: copy source files
+
+COPY . /src
+WORKDIR /src
+## Task: fetch project deps
+
+RUN go mod download
+
+## Task: build project
+
+ENV GOOS="linux"
+ENV GOARCH="amd64"
+ENV CGO_ENABLED="0"
+
+RUN go build -ldflags="-s -w" -o autorestsrv cmd/service.go 
+
+## Task: set permissions
+
+RUN chmod 0755 /src/autorestsrv
+
+## Task: runtime dependencies
+
+# hadolint ignore=DL3018
+RUN set -eux; \
+    apk add --no-progress --quiet --no-cache --upgrade --virtual .run-deps \
+        tzdata
+
+# hadolint ignore=DL3018,SC2183,DL4006
+RUN set -eu +x; \
+    apk add --no-progress --quiet --no-cache --upgrade ncurses; \
+    apk update --quiet; \
+    printf '%30s\n' | tr ' ' -; \
+    echo "RUNTIME DEPENDENCIES"; \
+    PKGNAME=$(apk info --depends .run-deps \
+        | sed '/^$/d;/depends/d' \
+        | sort -u ); \
+    printf '%s\n' "${PKGNAME}" \
+        | while IFS= read -r pkg; do \
+                apk info --quiet --description --no-network "${pkg}" \
+                | sed -n '/description/p' \
+                | sed -r "s/($(echo "${pkg}" | sed -r 's/\+/\\+/g'))-(.*)\s.*/\1=\2/"; \
+                done \
+        | tee -a /usr/share/rundeps; \
+    printf '%30s\n' | tr ' ' - 
+
+
+##### TARGET #####
+
+FROM alpine:3.11
+
+ARG RELEASE
+ENV IMG_VERSION="${RELEASE}"
+
+COPY --from=builder /src/autorestsrv /usr/local/bin/
+COPY --from=builder /src/configs/service.yaml /config/
+COPY --from=builder /usr/share/rundeps /usr/share/rundeps
+
+RUN set -eux; \
+    xargs -a /usr/share/rundeps apk add --no-progress --quiet --no-cache --upgrade --virtual .run-deps
+
+ENTRYPOINT ["/usr/local/bin/autorestsrv"]
+CMD ["--config","/config/service.yaml"]
+
+EXPOSE 8080 8443
+
+HEALTHCHECK --interval=30s --timeout=5s --retries=3 --start-period=10s \
+  CMD wget -q -T 5 --spider http://localhost:8080/health/health
+
+LABEL org.opencontainers.image.title="AutoRest-Service" \
+      org.opencontainers.image.description="MCS AutoRest Service" \
+      org.opencontainers.image.version="${IMG_VERSION}" \
+      org.opencontainers.image.source="https://github.com/willie68/AutoRestIoT.git" \
+      org.opencontainers.image.vendor="MCS (www.wk-music.de)" \
+      org.opencontainers.image.authors="Willie@mcs" \
+      maintainer="MCS" \
+      NAME="AutoRest-Service"
+

MessageService/api/adminapi.go

@@ -0,0 +1,348 @@
+package api
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/go-chi/chi"
+	"github.com/go-chi/render"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/config"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/dao"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/logging"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/model"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/worker"
+)
+
+var log logging.ServiceLogger
+
+const AdminPrefix = "admin"
+const BackendsPrefix = "backends"
+
+//AdminRoutes getting all routes for the config endpoint
+func AdminRoutes() *chi.Mux {
+	router := chi.NewRouter()
+	router.Mount("/tasks", TasksRoutes())
+	router.With(RoleCheck([]string{"admin"})).Get("/info", GetAdminInfoHandler)
+	router.With(RoleCheck([]string{"admin", "edit", "read"})).Get(fmt.Sprintf("/%s/", BackendsPrefix), GetAdminBackendsHandler)
+	router.With(RoleCheck([]string{"admin"})).Post(fmt.Sprintf("/%s/", BackendsPrefix), PostAdminBackendHandler)
+	router.With(RoleCheck([]string{"admin", "edit", "read"})).Get(fmt.Sprintf("/%s/{bename}", BackendsPrefix), GetAdminBackendHandler)
+	router.With(RoleCheck([]string{"admin"})).Delete(fmt.Sprintf("/%s/{bename}", BackendsPrefix), DeleteAdminBackendHandler)
+	router.With(RoleCheck([]string{"admin"})).Delete(fmt.Sprintf("/%s/{bename}/dropdata", BackendsPrefix), DeleteAdminBackendEndpoint)
+	router.With(RoleCheck([]string{"admin", "edit", "read"})).Get(fmt.Sprintf("/%s/{bename}/models", BackendsPrefix), GetAdminModelsHandler)
+	router.With(RoleCheck([]string{"admin", "edit", "read"})).Get(fmt.Sprintf("/%s/{bename}/models/{model}", BackendsPrefix), GetAdminModelHandler)
+	router.With(RoleCheck([]string{"admin"})).Get(fmt.Sprintf("/%s/{bename}/rules", BackendsPrefix), GetAdminRulesHandler)
+	router.With(RoleCheck([]string{"admin"})).Get(fmt.Sprintf("/%s/{bename}/rules/{rulename}", BackendsPrefix), GetAdminRulesRuleHandler)
+	router.With(RoleCheck([]string{"admin"})).Post(fmt.Sprintf("/%s/{bename}/rules/{rulename}/test", BackendsPrefix), PostAdminRulesRuleTestHandler)
+	router.With(RoleCheck([]string{"admin"})).Get(fmt.Sprintf("/%s/{bename}/datasources", BackendsPrefix), GetAdminDatasourcesHandler)
+	router.With(RoleCheck([]string{"admin"})).Get(fmt.Sprintf("/%s/{bename}/datasources/{datasource}", BackendsPrefix), GetAdminDatasourceHandler)
+	router.With(RoleCheck([]string{"admin"})).Get(fmt.Sprintf("/%s/{bename}/destinations", BackendsPrefix), GetAdminDestinationsHandler)
+	router.With(RoleCheck([]string{"admin"})).Get(fmt.Sprintf("/%s/{bename}/destinations/{destination}", BackendsPrefix), GetAdminDestinationHandler)
+	return router
+}
+
+type BackendInfo struct {
+	Name         string
+	Description  string
+	URL          string
+	Models       []string
+	Rules        []string
+	Datasources  []string
+	Destinations []string
+}
+
+// GetAdminInfoHandler getting server info
+func GetAdminInfoHandler(response http.ResponseWriter, request *http.Request) {
+	log.Infof("GET: path: %s", request.URL.Path)
+	info := model.JSONMap{}
+
+	info["backends"] = model.BackendList.Names()
+	info["modelcounter"] = worker.GetModelCount()
+	info["rules"] = worker.Rules.GetRulelist()
+	info["mqttClients"] = worker.GetMQTTClients()
+
+	render.JSON(response, request, info)
+}
+
+// GetAdminBackendsHandler create a new backend
+func GetAdminBackendsHandler(response http.ResponseWriter, request *http.Request) {
+	log.Infof("GET: path: %s", request.URL.Path)
+	names := model.BackendList.Names()
+	backendInfos := make([]BackendInfo, 0)
+	myconfig := config.Get()
+	for _, name := range names {
+		backend, _ := model.BackendList.Get(name)
+		modelNames := make([]string, 0)
+		for _, model := range backend.Models {
+			modelNames = append(modelNames, model.Name)
+		}
+		ruleNames := make([]string, 0)
+		for _, rule := range backend.Rules {
+			ruleNames = append(ruleNames, rule.Name)
+		}
+		datasourceNames := make([]string, 0)
+		for _, datasource := range backend.DataSources {
+			datasourceNames = append(datasourceNames, datasource.Name)
+		}
+		destinationNames := make([]string, 0)
+		for _, destination := range backend.Destinations {
+			destinationNames = append(destinationNames, destination.Name)
+		}
+		backendInfos = append(backendInfos, BackendInfo{
+			Name:         name,
+			Description:  backend.Description,
+			URL:          fmt.Sprintf("%s%s%s/", myconfig.ServiceURL, request.URL.Path, name),
+			Models:       modelNames,
+			Rules:        ruleNames,
+			Datasources:  datasourceNames,
+			Destinations: destinationNames,
+		})
+	}
+	render.JSON(response, request, backendInfos)
+}
+
+// GetAdminBackendHandler create a new backend
+func GetAdminBackendHandler(response http.ResponseWriter, request *http.Request) {
+	backendName := chi.URLParam(request, "bename")
+	log.Infof("GET: path: %s, be: %s", request.URL.Path, backendName)
+	backend, ok := model.BackendList.Get(backendName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	render.JSON(response, request, backend)
+}
+
+// DeleteAdminBackendHandler create a new backend
+func DeleteAdminBackendHandler(response http.ResponseWriter, request *http.Request) {
+	backendName := chi.URLParam(request, "bename")
+	log.Infof("DELETE: path: %s, be: %s", request.URL.Path, backendName)
+	backend, ok := model.BackendList.Get(backendName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	err := worker.DeregisterBackend(backendName)
+	if err != nil {
+		log.Alertf("%v\n", err)
+		render.Render(response, request, ErrInternalServer(err))
+		return
+	}
+
+	worker.DeleteBackend(backendName)
+
+	m := make(map[string]interface{})
+	m["backend"] = backend
+	m["msg"] = fmt.Sprintf("backend %s definition deleted. No data destroyed.", backendName)
+
+	render.JSON(response, request, m)
+}
+
+// PostAdminBackendHandler create a new backend
+func PostAdminBackendHandler(response http.ResponseWriter, request *http.Request) {
+	log.Infof("POST: path: %s", request.URL.Path)
+
+	data := &model.Backend{}
+	if err := render.Decode(request, data); err != nil {
+		render.Render(response, request, ErrInvalidRequest(err))
+		return
+	}
+
+	bemodel := *data
+	if !model.BackendList.Contains(bemodel.Backendname) {
+		render.Status(request, http.StatusCreated)
+	} else {
+		worker.DeregisterBackend(bemodel.Backendname)
+	}
+	bemodel, err := worker.PrepareBackend(bemodel)
+	if err != nil {
+		log.Alertf("%v\n", err)
+		render.Render(response, request, ErrInternalServer(err))
+		return
+	}
+	_, err = worker.StoreBackend(bemodel)
+	if err != nil {
+		log.Alertf("%v\n", err)
+		render.Render(response, request, ErrInternalServer(err))
+		return
+	}
+	err = worker.RegisterBackend(bemodel)
+	if err != nil {
+		log.Alertf("%v\n", err)
+		render.Render(response, request, ErrInternalServer(err))
+		return
+	}
+	render.JSON(response, request, bemodel)
+}
+
+//DeleteAdminBackendEndpoint delete a backend with data
+func DeleteAdminBackendEndpoint(response http.ResponseWriter, request *http.Request) {
+	backend := chi.URLParam(request, "bename")
+	log.Infof("DELETE: path: %s, be: %s", request.URL.Path, backend)
+
+	err := dao.GetStorage().DeleteBackend(backend)
+	if err != nil {
+		log.Alertf("%v\n", err)
+		render.Render(response, request, ErrInvalidRequest(err))
+		return
+	}
+
+	m := make(map[string]interface{})
+	m["backend"] = backend
+	m["msg"] = fmt.Sprintf("backend %s deleted. All data destroyed.", backend)
+
+	render.JSON(response, request, m)
+}
+
+// GetAdminModelsHandler getting all models of a backend
+func GetAdminModelsHandler(response http.ResponseWriter, request *http.Request) {
+	backendName := chi.URLParam(request, "bename")
+	log.Infof("GET: path: %s, be: %s", request.URL.Path, backendName)
+	backend, ok := model.BackendList.Get(backendName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	models := backend.Models
+	render.JSON(response, request, models)
+}
+
+// GetAdminModelHandler getting all models of a backend
+func GetAdminModelHandler(response http.ResponseWriter, request *http.Request) {
+	backendName := chi.URLParam(request, "bename")
+	modelName := chi.URLParam(request, "model")
+	log.Infof("GET: path: %s, be: %s, md: %s", request.URL.Path, backendName, modelName)
+	backend, ok := model.BackendList.Get(backendName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	myModel, ok := backend.GetModel(modelName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	render.JSON(response, request, myModel)
+}
+
+// GetAdminRulesHandler getting all rules of a backend
+func GetAdminRulesHandler(response http.ResponseWriter, request *http.Request) {
+	backendName := chi.URLParam(request, "bename")
+	log.Infof("GET: path: %s, be: %s", request.URL.Path, backendName)
+	backend, ok := model.BackendList.Get(backendName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	rules := backend.Rules
+	render.JSON(response, request, rules)
+}
+
+//GetAdminRulesRuleHandler getting all rules of a backend
+func GetAdminRulesRuleHandler(response http.ResponseWriter, request *http.Request) {
+	backendName := chi.URLParam(request, "bename")
+	ruleName := chi.URLParam(request, "rulename")
+	log.Infof("GET: path: %s, be: %s", request.URL.Path, backendName)
+	backend, ok := model.BackendList.Get(backendName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	rule, ok := backend.GetRule(ruleName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	render.JSON(response, request, rule)
+}
+
+//PostAdminRulesRuleTestHandler getting all rules of a backend
+func PostAdminRulesRuleTestHandler(response http.ResponseWriter, request *http.Request) {
+	backendName := chi.URLParam(request, "bename")
+	ruleName := chi.URLParam(request, "rulename")
+	log.Infof("POST: path: %s, be: %s", request.URL.Path, backendName)
+	backend, ok := model.BackendList.Get(backendName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	_, ok = backend.GetRule(ruleName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	body, err := ioutil.ReadAll(request.Body)
+	if err != nil {
+		render.Render(response, request, ErrInvalidRequest(err))
+		return
+	}
+	transformedJson, err := worker.Rules.TransformJSON(backendName, ruleName, body)
+	if err != nil {
+		render.Render(response, request, ErrInvalidRequest(err))
+		return
+	}
+	response.Header().Add("Content-Type", "application/json")
+	response.Write(transformedJson)
+}
+
+// GetAdminDatasourcesHandler getting all datasources of a backend
+func GetAdminDatasourcesHandler(response http.ResponseWriter, request *http.Request) {
+	backendName := chi.URLParam(request, "bename")
+	log.Infof("GET: path: %s, be: %s", request.URL.Path, backendName)
+	backend, ok := model.BackendList.Get(backendName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	datasources := backend.DataSources
+	render.JSON(response, request, datasources)
+}
+
+// GetAdminDatasourceHandler getting all datasources of a backend
+func GetAdminDatasourceHandler(response http.ResponseWriter, request *http.Request) {
+	backendName := chi.URLParam(request, "bename")
+	datasourceName := chi.URLParam(request, "datasource")
+	log.Infof("GET: path: %s, be: %s, src: %s", request.URL.Path, backendName, datasourceName)
+	backend, ok := model.BackendList.Get(backendName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	datasource, ok := backend.GetDatasource(datasourceName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	render.JSON(response, request, datasource)
+}
+
+// GetAdminDestinationsHandler getting all datasources of a backend
+func GetAdminDestinationsHandler(response http.ResponseWriter, request *http.Request) {
+	backendName := chi.URLParam(request, "bename")
+	log.Infof("GET: path: %s, be: %s", request.URL.Path, backendName)
+	backend, ok := model.BackendList.Get(backendName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	destinations := backend.Destinations
+	render.JSON(response, request, destinations)
+}
+
+// GetAdminDestinationHandler getting all datasources of a backend
+func GetAdminDestinationHandler(response http.ResponseWriter, request *http.Request) {
+	backendName := chi.URLParam(request, "bename")
+	destinationName := chi.URLParam(request, "destination")
+	log.Infof("GET: path: %s, be: %s, src: %s", request.URL.Path, backendName, destinationName)
+	backend, ok := model.BackendList.Get(backendName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	destination, ok := backend.GetDestination(destinationName)
+	if !ok {
+		render.Render(response, request, ErrNotFound)
+		return
+	}
+	render.JSON(response, request, destination)
+}

MessageService/api/basicauth.go

@@ -0,0 +1,38 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/dao"
+)
+
+// BasicAuth implements a simple middleware handler for adding basic http auth to a route.
+func BasicAuth(realm string) func(next http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			user, pass, ok := r.BasicAuth()
+			if !ok {
+				basicAuthFailed(w, realm)
+				return
+			}
+			idm := dao.GetIDM()
+			salt, ok := idm.GetSalt(user)
+			if !ok {
+				basicAuthFailed(w, realm)
+				return
+			}
+			pass = dao.BuildPasswordHash(pass, salt)
+			if !idm.CheckUser(user, pass) {
+				basicAuthFailed(w, realm)
+				return
+			}
+			next.ServeHTTP(w, r)
+		})
+	}
+}
+
+func basicAuthFailed(w http.ResponseWriter, realm string) {
+	w.Header().Add("WWW-Authenticate", fmt.Sprintf(`Basic realm="%s"`, realm))
+	w.WriteHeader(http.StatusUnauthorized)
+}

MessageService/api/filesapi.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
-	"github.com/willie68/AutoRestIoT/dao"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/dao"
 )
 
 //FilesRoutes getting all routes for the config endpoint

MessageService/api/modelapi.go

@@ -8,10 +8,10 @@ import (
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
-	"github.com/willie68/AutoRestIoT/dao"
-	"github.com/willie68/AutoRestIoT/internal"
-	"github.com/willie68/AutoRestIoT/model"
-	"github.com/willie68/AutoRestIoT/worker"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/dao"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/internal"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/model"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/worker"
 )
 
 //DeleteRefHeader header key for the delete reference options

MessageService/api/rolehandler.go

@@ -4,7 +4,7 @@ import (
 	"net/http"
 
 	"github.com/go-chi/render"
-	"github.com/willie68/AutoRestIoT/dao"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/dao"
 )
 
 //RoleCheck implements a simple middleware handler for adding a role check to a route.

MessageService/api/tasksapi.go

@@ -5,8 +5,8 @@ import (
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
-	"github.com/willie68/AutoRestIoT/dao"
-	"github.com/willie68/AutoRestIoT/worker"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/dao"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/worker"
 )
 
 //AdminRoutes getting all routes for the config endpoint

MessageService/api/userapi.go

@@ -6,8 +6,8 @@ import (
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/render"
-	"github.com/willie68/AutoRestIoT/dao"
-	"github.com/willie68/AutoRestIoT/model"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/dao"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/model"
 )
 
 // UsersRoutes routes to the user interface

MessageService/architectual.md

@@ -0,0 +1,598 @@
+# Message Service #
+
+## Einleitung
+
+## Konfiguration des Service
+
+```yaml
+# port of the http server
+port: 8080 
+# port of the https server
+sslport: 8443
+# this is the servicURL from outside
+serviceURL: http://127.0.0.1:8080
+# this is the registry URL from inside this service for consul as service registry
+registryURL: 
+# this is the system id of this service. services in a cluster mode should have the same system id.
+systemID: autorest-srv
+#sercret file for storing usernames and passwords
+secretfile: /tmp/storage/config/secret.yaml
+#where the configuration files of the backends are
+backendpath: configs/backends
+#allow data saving without a registered backend
+allowAnonymousBackend: true
+
+# configuration of the gelf logging server
+logging:
+    gelf-url: 
+    gelf-port: 
+
+# healthcheck configuration
+healthcheck:
+    # automatically check the health of this service every ## seconds
+    period: 30
+# background tasks configuration
+backgroundtasks:
+    deleteOrphanedFiles: true # automatically delete orphaned files
+    period: 86400  #once a day
+
+# configuration of the mongo storage
+mongodb:
+    host: 127.0.0.1 #mongo host ip (comma seperated ip's with a cluster installation )
+    port: 27017 # mongo host port
+    username: #username for the database (should be at last dbadmin, and read/write access)
+    password: #password for the user
+    authdb: backend1 # database to authenticate against
+    database: backend1 # database for the data
+
+```
+
+Die Secrets zu diesem Service, im speziellen username und Passwort zur MongoDB werden in einer speziellen Datei secrets.yaml abgelegt. Diese wird über den Configeintrag `secretfile` bestimmt.
+
+Inhalt:
+
+```yaml
+mongodb:
+    username: backend1
+    password: backend1
+```
+
+
+
+## Storage
+
+Als Storage wird derzeit nur MongoDB unterstützt. 
+
+Bei der Mongo Storage Implementierung werden die verschiedenen Backends allerdings in einer Datenbank abgelegt. Einzelne Modelle werden in jeweils einer Collection abgelegt.  Der Collectionname besteht aus dem Backendnamen  "." und dem Modellnamen. 
+
+### Hint
+
+Um eine neue Mongodatenbank anzulegen, müssen folgende Kommandos auf der Mongo Console ausgeführt werden:
+
+```json
+#create a new db named backend1
+use backend1
+#create a db admin on db backend1 with user backend1 with password backend1
+db.createUser({ user: "backend1", pwd: "backend1", roles: [ "readWrite", "dbAdmin", { role: "dbOwner", db: "backend1" } ]})
+
+```
+
+
+
+## User
+
+Folgende User mit folgenden Rollen werden automatisch angelegt:
+
+- Admin, pwd: admin, roles: admin
+- Editor, pwd: editor, roles: edit
+- guest, pwd: guest, roles: read
+
+## REST Interface
+
+Hier nun folgt die Beschreibung des REST Interfaces. Beispielhafte REST Calls sind als Postman Collection im Ordner test/postman vorhanden. 
+
+Beispielhaft werden hier alle Calls als Calls auf den lokalen Server (127.0.0.1) mit dem Port 9443 bereitgestellt. Bei einer anderen Serverinstanz bitte entsprechend ändern.
+
+### Admin API
+
+Security: Ja, Authentifizierung derzeit als BasicAuth.
+Role: admin
+
+zus. Header:
+
+**X-mcs-system**: autorest-srv  (Konfigurationseinstellung: systemID)
+
+**X-mcs-apikey**: {uuid} 
+Wird beim Starten des Servers auf der Konsole ausgegeben. 
+
+```
+...
+2020/04/29 08:43:04 systemid: autorest-srv
+2020/04/29 08:43:04 apikey: 5854d123dd25f310395954f7c450171c
+2020/04/29 08:43:04 ssl: true
+...
+```
+
+
+
+#### Liste alle Backends
+
+**Request**: **GET**: https://127.0.0.1:9443/api/v1/admin/backends
+
+**Beschreibung**: Liefert eine Liste mit allen Backenddefinitioninformationen. D.h. pro Backend werden nur der Name, die Beschreibung und die URL auf die Definition ausgeliefert.
+
+**Request**: **GET**: https://127.0.0.1:9443/api/v1/admin/backends
+
+**Response**:
+
+```JSON
+[
+    {
+        "Name": "sensors",
+        "Description": "sensor model für storing and retrieving sensor data",
+        "URL": "https://127.0.0.1:9443/api/v1/admin/backends/sensors/"
+    },
+    {
+        "Name": "mybe",
+        "Description": "",
+        "URL": "https://127.0.0.1:9443/api/v1/admin/backends/mybe/"
+    }
+]
+```
+
+#### Definition eines Backends
+
+**GET**: https://127.0.0.1:9443/api/v1/admin/backends/{backendname}/
+
+**Beschreibung**: Liefert die Definition eines Backends.
+
+**Request**: https://127.0.0.1:9443/api/v1/admin/backends/sensors/
+
+**Response**:
+
+```JSON
+{
+    "backendname": "sensors",
+    "description": "sensor model für storing and retrieving sensor data",
+    "models": [
+        {
+            "name": "temperatur",
+            "description": "",
+            "fields": [
+                {
+                    "name": "temperatur",
+                    "type": "float",
+                    "mandatory": false,
+                    "collection": false
+                },
+                {
+                    "name": "source",
+                    "type": "string",
+                    "mandatory": false,
+                    "collection": false
+                }
+            ],
+            "indexes": null
+        }
+    ],
+    "datasources": [
+        {
+            "name": "temp_wohnzimmer",
+            "type": "mqtt",
+            "destination": "temperatur",
+            "rule": "tasmota_ds18b20",
+            "config": {
+                "broker": "127.0.0.1:1883",
+                "topic": "stat/temperatur/wohnzimmer",
+                "payload": "application/json",
+                "username": "temp",
+                "password": "temp",
+                "addTopicAsAttribute": "topic",
+                "simpleValueAttribute": ""
+            }
+        },
+        {
+            "name": "temp_kueche",
+            "type": "mqtt",
+            "destination": "temperatur",
+            "rule": "tasmota_ds18b20",
+            "config": {
+                "broker": "127.0.0.1:1883",
+                "topic": "tele/tasmota_63E6F8/SENSOR",
+                "payload": "application/json",
+                "username": "temp",
+                "password": "temp",
+                "addTopicAsAttribute": "topic",
+                "simpleValueAttribute": ""
+            }
+        }
+    ],
+    "rules": [
+        {
+            "name": "tasmota_ds18b20",
+            "description": "transforming the tasmota json structure of the DS18B20 into my simple structure",
+            "transform": [
+                {
+                    "operation": "shift",
+                    "spec": {
+                        "TempUnit": "TempUnit",
+                        "Temperature": "DS18B20.Temperature"
+                    }
+                }
+            ]
+        },
+        {
+            "name": "hm_temp_simple",
+            "description": "handle homematic temperatur rightly",
+            "transform": [
+                {
+                    "operation": "shift",
+                    "spec": {
+                        "Datetime": "ts",
+                        "Temperature": "val",
+                        "Timestamp": "ts"
+                    }
+                },
+                {
+                    "operation": "default",
+                    "spec": {
+                        "TempUnit": "°C"
+                    }
+                },
+                {
+                    "operation": "timestamp",
+                    "spec": {
+                        "Datetime": {
+                            "inputFormat": "$unixext",
+                            "outputFormat": "2006-01-02T15:04:05-0700"
+                        }
+                    }
+                }
+            ]
+        }
+    ]
+}
+```
+
+#### Neues Backends anlegen
+
+**POST**: https://127.0.0.1:9443/api/v1/admin/backends/
+
+**Beschreibung**: Liefert die Definition eines Backends.
+
+**Request**: https://127.0.0.1:9443/api/v1/admin/backends/sensors/
+
+**Response**: **Not implemented Yet**
+
+#### Daten eines Backends löschen
+
+**DELETE**: https://127.0.0.1:9443/api/v1/admin/backends/
+
+**Beschreibung**: Löscht alle Daten eines Backends.
+
+**Request**: https://127.0.0.1:9443/api/v1/admin/backends/sensors/dropdata
+
+**Response**: 
+
+```json
+{
+    "backend": "sensors",
+    "msg": "backend sensors deleted. All data destroyed."
+}
+```
+
+#### Liste aller Modelle eines Backends 
+
+**GET**: https://127.0.0.1:9443/api/v1/admin/backends/{backendname}/models
+
+**Beschreibung**: Liefert eine Liste aller Modelle eines Backends.
+
+**Request**: https://127.0.0.1:9443/api/v1/admin/backends/sensors/models
+
+**Response**: 
+
+```json
+[
+    {
+        "name": "temperatur",
+        "description": "",
+        "fields": [
+            {
+                "name": "temperatur",
+                "type": "float",
+                "mandatory": false,
+                "collection": false
+            },
+            {
+                "name": "source",
+                "type": "string",
+                "mandatory": false,
+                "collection": false
+            }
+        ],
+        "indexes": null
+    }
+]
+```
+
+#### Liste aller Datenquellen eines Backends 
+
+**GET**: https://127.0.0.1:9443/api/v1/admin/backends/{backendname}/datasources
+
+**Beschreibung**: Liefert eine Liste aller Datenquellen eines Backends.
+
+**Request**: https://127.0.0.1:9443/api/v1/admin/backends/sensors/datasources
+
+**Response**: 
+
+```json
+[
+    {
+        "name": "temp_wohnzimmer",
+        "type": "mqtt",
+        "destination": "temperatur",
+        "rule": "tasmota_ds18b20",
+        "config": {
+            "broker": "127.0.0.1:1883",
+            "topic": "stat/temperatur/wohnzimmer",
+            "payload": "application/json",
+            "username": "temp",
+            "password": "temp",
+            "addTopicAsAttribute": "topic",
+            "simpleValueAttribute": ""
+        }
+    },
+    {
+        "name": "temp_kueche",
+        "type": "mqtt",
+        "destination": "temperatur",
+        "rule": "tasmota_ds18b20",
+        "config": {
+            "broker": "127.0.0.1:1883",
+            "topic": "tele/tasmota_63E6F8/SENSOR",
+            "payload": "application/json",
+            "username": "temp",
+            "password": "temp",
+            "addTopicAsAttribute": "topic",
+            "simpleValueAttribute": ""
+        }
+    }
+]
+```
+
+#### Liste aller Transformationsregeln eines Backends 
+
+**GET**: https://127.0.0.1:9443/api/v1/admin/backends/{backendname}/rules
+
+**Beschreibung**: Liefert eine Liste aller Transformationsregelen eines Backends.
+
+**Request**: https://127.0.0.1:9443/api/v1/admin/backends/sensors/rules
+
+**Response**: 
+
+```json
+[
+    {
+        "name": "tasmota_ds18b20",
+        "description": "transforming the tasmota json structure of the DS18B20 into my simple structure",
+        "transform": [
+            {
+                "operation": "shift",
+                "spec": {
+                    "TempUnit": "TempUnit",
+                    "Temperature": "DS18B20.Temperature"
+                }
+            }
+        ]
+    },
+    {
+        "name": "hm_temp_simple",
+        "description": "handle homematic temperatur rightly",
+        "transform": [
+            {
+                "operation": "shift",
+                "spec": {
+                    "Datetime": "ts",
+                    "Temperature": "val",
+                    "Timestamp": "ts"
+                }
+            },
+            {
+                "operation": "default",
+                "spec": {
+                    "TempUnit": "°C"
+                }
+            },
+            {
+                "operation": "timestamp",
+                "spec": {
+                    "Datetime": {
+                        "inputFormat": "$unixext",
+                        "outputFormat": "2006-01-02T15:04:05-0700"
+                    }
+                }
+            }
+        ]
+    }
+]
+```
+
+#### Definition einer Transformationsregel eines Backends 
+
+**GET**: https://127.0.0.1:9443/api/v1/admin/backends/{backendname}/rules/{rulename}
+
+**Beschreibung**: Definition einer Transformationsregel eines Backends.
+
+**Request**: https://127.0.0.1:9443/api/v1/admin/backends/sensors/rules/hm_temp_simple
+
+**Response**: 
+
+```json
+{
+    "name": "hm_temp_simple",
+    "description": "handle homematic temperatur rightly",
+    "transform": [
+        {
+            "operation": "shift",
+            "spec": {
+                "Datetime": "ts",
+                "Temperature": "val",
+                "Timestamp": "ts"
+            }
+        },
+        {
+            "operation": "default",
+            "spec": {
+                "TempUnit": "°C"
+            }
+        },
+        {
+            "operation": "timestamp",
+            "spec": {
+                "Datetime": {
+                    "inputFormat": "$unixext",
+                    "outputFormat": "2006-01-02T15:04:05-0700"
+                }
+            }
+        }
+    ]
+}
+```
+
+#### Testen einer Transformationsregel eines Backends 
+
+**POST**: https://127.0.0.1:9443/api/v1/admin/backends/{backendname}/rules/{rulename}/test
+
+**Beschreibung**: Testen einer Transformationsregel eines Backends.
+
+**Request**: https://127.0.0.1:9443/api/v1/admin/backends/sensors/rules/hm_temp_simple/test
+
+Payload:
+
+```json
+{
+    "val": 22.8,
+    "ts": 1588142598973,
+    "lc": 1588142598973
+}
+```
+
+**Response**: 
+
+```json
+{
+    "Timestamp": 1588142598973,
+    "Datetime": "2020-04-29T08:43:18+0200",
+    "Temperature": 22.8,
+    "TempUnit": "°C"
+}
+```
+
+### Tasks API
+
+Security: Ja
+
+#### Taskliste
+
+Request**: **GET**: https://127.0.0.1:9443/api/v1/admin/tasks
+
+**Beschreibung**: Liste der Service Tasks
+
+**Security role**: admin
+
+**Request**: **GET**: https://127.0.0.1:9443/api/v1/admin/tasks
+
+**Response**:
+
+```json
+{
+    "count": 1,
+    "data": [
+        {
+            "_id": "5eca0c72cfe02b227085e242",
+            "tdata": "reporting",
+            "tfile": "1234456789",
+            "tstatus": "finished",
+            "ttype": "report"
+        }
+    ],
+    "found": 1,
+    "limit": 0,
+    "offset": 0,
+    "query": ""
+}
+```
+
+#### Starten eines Servicetask
+
+**Request**: **POST**: https://127.0.0.1:9443/api/v1/admin/tasks
+
+**Beschreibung**: Starten eines Service tasks
+
+**Security role**: admin
+
+**Request**: **POST**: https://127.0.0.1:9443/api/v1/admin/tasks/
+
+​	**Payload**: 
+
+```json
+{
+  	"ttype": "report",
+    "tdata": "[optional data for the task, if necessary]"
+}
+```
+
+**Response**:
+
+```JSON
+{
+    "tasksid": "5ea92df4d015d95201f6b4b8"
+}
+```
+
+​	**Headers**: `Location: /api/v1/admin/tasks/5ea92df4d015d95201f6b4b8`
+
+### Files API
+
+Security: Ja
+
+
+#### Upload einer Datei
+
+**Request**: **POST**: https://127.0.0.1:9443/api/v1/files/{backendname}/
+
+**Beschreibung**: Upload einer Datei auf den Server. Dateien dürfen nicht größer sein als 10MB.
+
+**Security role**: edit
+
+**Request**: **POST**: https://127.0.0.1:9443/api/v1/files/sensors/
+
+​	**Payload**: Http Formbased File Upload. Name des Formfeldes: file
+
+**Response**:
+
+```JSON
+{
+    "fileid": "5ea92df4d015d95201f6b4b8",
+    "filename": "readme.md"
+}
+```
+
+​	**Headers**: `Location: /api/v1/files/sensors/5ea92df4d015d95201f6b4b8`
+
+#### Download einer Datei
+
+**Request**: **GET**: https://127.0.0.1:9443/api/v1/files/{backendname}/{fileid}
+
+**Beschreibung**: Download einer Datei vom Server.
+
+**Security role**: read
+
+**Request**: **GET**: https://127.0.0.1:9443/api/v1/files/sensors/5ea92df4d015d95201f6b4b8
+
+**Response**:
+
+Die Datei als Download.
+
+
+

MessageService/cmd/adminapi_test.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"io/ioutil"
+	"net/http"
+	"testing"
+)
+
+func TestGetBackend(t *testing.T) {
+	url := baseSslURL + restEndpoints + "admin/backends/mybe/"
+	resp, err := getGetRequest(url, AdminUser)
+	if err != nil {
+		t.Errorf("Error getting response. %v", err)
+		return
+	}
+	defer resp.Body.Close()
+
+	bodyText, err := ioutil.ReadAll(resp.Body)
+	s := string(bodyText)
+	t.Logf("response: url: %s, response: %s", url, s)
+
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("Wrong statuscode: %d", resp.StatusCode)
+		return
+	}
+
+	t.Log("Get Backend OK.")
+}

MessageService/cmd/service.go

@@ -15,18 +15,18 @@ import (
 	"strings"
 	"time"
 
-	api "github.com/willie68/AutoRestIoT/api"
-	"github.com/willie68/AutoRestIoT/dao"
-	"github.com/willie68/AutoRestIoT/health"
-	"github.com/willie68/AutoRestIoT/model"
-	"github.com/willie68/AutoRestIoT/worker"
 	"gopkg.in/yaml.v3"
+	api "wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/api"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/dao"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/health"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/model"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/worker"
 
-	"github.com/willie68/AutoRestIoT/internal/crypt"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/internal/crypt"
 
 	consulApi "github.com/hashicorp/consul/api"
-	config "github.com/willie68/AutoRestIoT/config"
-	"github.com/willie68/AutoRestIoT/logging"
+	config "wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/config"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/logging"
 
 	"github.com/go-chi/chi"
 	"github.com/go-chi/chi/middleware"

MessageService/cmd/service_test.go

@@ -8,7 +8,7 @@ import (
 	"net/http"
 	"strings"
 
-	api "github.com/willie68/AutoRestIoT/api"
+	api "wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/api"
 )
 
 const baseURL = "http://127.0.0.1:9080"

MessageService/cmd/userapi_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/willie68/AutoRestIoT/model"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/model"
 )
 
 func TestGetUsers(t *testing.T) {

MessageService/config/config.go

@@ -0,0 +1,57 @@
+package config
+
+// Config our service configuration
+type Config struct {
+	//port of the http server
+	Port int `yaml:"port"`
+	//port of the https server
+	Sslport int `yaml:"sslport"`
+	//this is the url how to connect to this service from outside
+	ServiceURL string `yaml:"serviceURL"`
+	//this is the url where to register this service
+	RegistryURL string `yaml:"registryURL"`
+	//this is the url where to register this service
+	SystemID    string `yaml:"systemID"`
+	BackendPath string `yaml:"backendpath"`
+
+	SecretFile string `yaml:"secretfile"`
+
+	WebRoot string `yaml:"webRoot"`
+
+	AllowAnonymousBackend bool `yaml:"allowAnonymousBackend"`
+
+	Logging Logging `yaml:"logging"`
+
+	HealthCheck HealthCheck `yaml:"healthcheck"`
+
+	BackgroundTasks BackgroundTasks `yaml:"backgroundtasks"`
+
+	MongoDB MongoDB `yaml: "mongodb"`
+}
+
+//Logging configuration for the logging system (At the moment only for the gelf logger)
+type Logging struct {
+	Gelfurl  string `yaml:"gelf-url"`
+	Gelfport int    `yaml:"gelf-port"`
+}
+
+//HealthCheck configuration for the healthcheck system
+type HealthCheck struct {
+	Period int `yaml:"period"`
+}
+
+//BackgroundTasks configuration for the background tasks system
+type BackgroundTasks struct {
+	Period              int  `yaml:"period"`
+	DeleteOrphanedFiles bool `yaml:"deleteOrphanedFiles"`
+}
+
+//MongoDB configuration for the mongodb stoirage
+type MongoDB struct {
+	Host     string `yaml:"host"`
+	Port     int    `yaml:"port"`
+	Username string `yaml:"username"`
+	Password string `yaml:"password"`
+	AuthDB   string `yaml:"authdb"`
+	Database string `yaml:"database"`
+}

MessageService/config/loader.go

@@ -5,8 +5,8 @@ import (
 	"io/ioutil"
 	"os"
 
-	"github.com/willie68/AutoRestIoT/logging"
 	"gopkg.in/yaml.v3"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/logging"
 )
 
 var config = Config{

MessageService/configs/backends/effects.yaml

@@ -0,0 +1,65 @@
+backendname: effects
+description: Willies World Effects Database
+models:
+    - name: effects
+      fields: 
+        - name: manufacturer
+          type: string
+          mandatory: true
+          collection: false
+        - name: effectType
+          type: string
+          mandatory: true
+          collection: false
+        - name: model
+          type: string
+          mandatory: true
+          collection: false
+        - name: comment
+          type: string
+          mandatory: true
+          collection: false
+        - name: image
+          type: file
+          mandatory: false
+          collection: false
+        - name: connector
+          type: string
+          mandatory: false
+          collection: false
+        - name: voltage
+          type: string
+          mandatory: false
+          collection: false
+        - name: current
+          type: string
+          mandatory: false
+          collection: false
+      indexes:
+        - name: $fulltext
+          fields:
+            - manufacturer
+            - effectType
+            - model
+            - comment
+            - voltage
+            - current
+    - name: effectTypes
+      fields: 
+        - name: typeName
+          type: string
+          mandatory: true
+          collection: false
+        - name: nls
+          type: map
+          mandatory: false
+          collection: false
+        - name: typeImage
+          type: file
+          mandatory: false
+          collection: false
+      indexes:
+        - name: $fulltext
+          fields:
+            - typeName
+            - nls

MessageService/dao/idm.go

@@ -12,9 +12,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/willie68/AutoRestIoT/internal/slicesutils"
-	"github.com/willie68/AutoRestIoT/model"
 	"golang.org/x/crypto/pbkdf2"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/internal/slicesutils"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/model"
 )
 
 //IDM the idm cache struct with users and salts

MessageService/dao/mongodao.go

@@ -10,12 +10,12 @@ import (
 	"strings"
 	"time"
 
-	"github.com/willie68/AutoRestIoT/config"
-	"github.com/willie68/AutoRestIoT/internal"
-	"github.com/willie68/AutoRestIoT/internal/crypt"
-	"github.com/willie68/AutoRestIoT/internal/slicesutils"
-	"github.com/willie68/AutoRestIoT/logging"
-	"github.com/willie68/AutoRestIoT/model"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/config"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/internal"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/internal/crypt"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/internal/slicesutils"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/logging"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/model"
 
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"

MessageService/dao/storageDao.go

@@ -3,7 +3,7 @@ package dao
 import (
 	"io"
 
-	"github.com/willie68/AutoRestIoT/model"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/model"
 )
 
 //FulltextIndexName name of the index containing fulltext data

MessageService/deployments/build.cmd

@@ -0,0 +1,2 @@
+@echo off
+go build -ldflags="-s -w" -o autorest-srv.exe cmd/service.go

MessageService/deployments/buildDocker.cmd

@@ -0,0 +1,3 @@
+@echo off
+docker build ./ -t mcs/autorestiot:V1
+docker run --name autorestiot -p 9443:9443 -p 9080:9080 mcs/autorestiot:V1

MessageService/go.mod

@@ -1,4 +1,4 @@
-module wkla.no-ip.biz/gogs/Willie/MsgService/
+module wkla.no-ip.biz/gogs/Willie/MsgService/MessageService
 
 go 1.14
 

MessageService/health/health.go

@@ -6,8 +6,8 @@ import (
 	"time"
 
 	"github.com/go-chi/chi"
-	"github.com/willie68/AutoRestIoT/dao"
-	"github.com/willie68/AutoRestIoT/logging"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/dao"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/logging"
 )
 
 // global storage definition

MessageService/internal/constants.go

@@ -0,0 +1,15 @@
+package internal
+
+const AttributeID = "_id"
+const AttributeOwner = "_owner"
+const AttributeCreated = "_created"
+const AttributeModified = "_modified"
+
+func TrimQuotes(s string) string {
+	if len(s) >= 2 {
+		if s[0] == '"' && s[len(s)-1] == '"' {
+			return s[1 : len(s)-1]
+		}
+	}
+	return s
+}

MessageService/internal/slicesutils/slicesutils_test.go

@@ -3,7 +3,7 @@ package slicesutils_test
 import (
 	"testing"
 
-	"github.com/willie68/AutoRestIoT/internal/slicesutils"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/internal/slicesutils"
 )
 
 func TestContains(t *testing.T) {

MessageService/model/backend.go

@@ -0,0 +1,273 @@
+package model
+
+import (
+	"errors"
+	"sort"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+//Backend struct for definition of a backend
+type Backend struct {
+	Backendname  string        `yaml:"backendname" json:"backendname"`
+	Description  string        `yaml:"description" json:"description"`
+	Models       []Model       `yaml:"models" json:"models"`
+	DataSources  []DataSource  `yaml:"datasources" json:"datasources"`
+	Rules        []Rule        `yaml:"rules" json:"rules"`
+	Destinations []Destination `yaml:"destinations" json:"destinations"`
+}
+
+//Model definition of a single model
+type Model struct {
+	Name        string  `yaml:"name" json:"name"`
+	Description string  `yaml:"description" json:"description"`
+	Fields      []Field `yaml:"fields" json:"fields"`
+	Indexes     []Index `yaml:"indexes" json:"indexes"`
+}
+
+//FieldTypeString field type string
+const FieldTypeString = "string"
+
+//FieldTypeInt field type integer
+const FieldTypeInt = "int"
+
+//FieldTypeFloat field type float
+const FieldTypeFloat = "float"
+
+//FieldTypeTime field type time
+const FieldTypeTime = "time"
+
+//FieldTypeBool field type bool
+const FieldTypeBool = "bool"
+
+//FieldTypeMap field type map
+const FieldTypeMap = "map"
+
+//FieldTypeFile field type file
+const FieldTypeFile = "file"
+
+//Field definition of a field
+type Field struct {
+	Name       string `yaml:"name" json:"name"`
+	Type       string `yaml:"type" json:"type"`
+	Mandatory  bool   `yaml:"mandatory" json:"mandatory"`
+	Collection bool   `yaml:"collection" json:"collection"`
+}
+
+//Index definition of an index
+type Index struct {
+	Name   string   `yaml:"name" json:"name"`
+	Unique bool     `yaml:"unique" json:"unique"`
+	Fields []string `yaml:"fields" json:"fields"`
+}
+
+//DataSource Definition of a datasource
+type DataSource struct {
+	Name         string      `yaml:"name" json:"name"`
+	Type         string      `yaml:"type" json:"type"`
+	Destinations []string    `yaml:"destinations" json:"destinations"`
+	Rule         string      `yaml:"rule" json:"rule"`
+	Config       interface{} `yaml:"config" json:"config"`
+}
+
+type Destination struct {
+	Name   string      `yaml:"name" json:"name"`
+	Type   string      `yaml:"type" json:"type"`
+	Config interface{} `yaml:"config" json:"config"`
+}
+
+type Rule struct {
+	Name        string      `yaml:"name" json:"name"`
+	Description string      `yaml:"description" json:"description"`
+	Transform   interface{} `yaml:"transform" json:"transform"`
+}
+
+//ErrModelDefinitionNotFound model definition not found
+var ErrModelDefinitionNotFound = errors.New("model defintion not found")
+
+//BackendList list of all definied backends
+var BackendList = NewBackends()
+
+//Backends backend list object
+type Backends struct {
+	bs map[string]Backend
+}
+
+//NewBackends creating a new Backend list
+func NewBackends() Backends {
+	b := Backends{
+		bs: make(map[string]Backend),
+	}
+	return b
+}
+
+//Names getting all backendnames
+func (m *Backends) Names() []string {
+	names := make([]string, 0)
+	for name := range m.bs {
+		names = append(names, name)
+	}
+	sort.Slice(names, func(i, j int) bool {
+		return names[i] < names[j]
+	})
+	return names
+}
+
+//Contains checking if the manufacturer name is present in the list of manufacturers
+func (m *Backends) Contains(name string) bool {
+	for k := range m.bs {
+		if k == name {
+			return true
+		}
+	}
+	return false
+}
+
+//Add adding a new manufacturer to the list
+func (m *Backends) Add(backend Backend) string {
+	m.bs[backend.Backendname] = backend
+	return backend.Backendname
+}
+
+//Remove remove a single tag
+func (m *Backends) Remove(name string) {
+	if m.Contains(name) {
+		delete(m.bs, name)
+	}
+}
+
+//Get getting a tag
+func (m *Backends) Get(name string) (Backend, bool) {
+	for k, be := range m.bs {
+		if k == name {
+			return be, true
+		}
+	}
+	return Backend{}, false
+}
+
+//Clear clearing the list
+func (m *Backends) Clear() {
+	m.bs = make(map[string]Backend)
+}
+
+//IsValidDatamodel checking if a data model is valid
+func (b *Backend) IsValidDatamodel(model string, data JSONMap) bool {
+	return true
+}
+
+//GetReferencedFiles getting a list of ids of referenced files
+func (b *Backend) GetReferencedFiles(modelname string, data JSONMap) ([]string, error) {
+	model, ok := b.GetModel(modelname)
+	if !ok {
+		return nil, ErrModelDefinitionNotFound
+	}
+	files := make([]string, 0)
+	for _, field := range model.Fields {
+		if field.Type == FieldTypeFile {
+			dataValue := data[field.Name]
+			if dataValue != nil {
+				switch v := dataValue.(type) {
+				case primitive.A:
+					values := v
+					for _, d := range values {
+						files = append(files, d.(string))
+					}
+				case []interface{}:
+					values := v
+					for _, d := range values {
+						files = append(files, d.(string))
+					}
+				case []string:
+					values := v
+					for _, d := range values {
+						files = append(files, d)
+					}
+				case string:
+					files = append(files, v)
+				}
+
+			}
+		}
+	}
+
+	return files, nil
+}
+
+//GetModel getting a model definition from the backend definition
+func (m *Backends) GetModel(route Route) (Model, bool) {
+	backend, ok := m.Get(route.Backend)
+	if !ok {
+		return Model{}, false
+	}
+	return backend.GetModel(route.Model)
+}
+
+//GetModel getting a model definition from the backend definition
+func (b *Backend) GetModel(modelname string) (Model, bool) {
+	for _, model := range b.Models {
+		if model.Name == modelname {
+			return model, true
+		}
+	}
+	return Model{}, false
+}
+
+//GetDatasource getting a datasource definition from the backend definition
+func (b *Backend) GetDatasource(datasourcename string) (DataSource, bool) {
+	for _, datasource := range b.DataSources {
+		if datasource.Name == datasourcename {
+			return datasource, true
+		}
+	}
+	return DataSource{}, false
+}
+
+//GetDestination getting a datasource definition from the backend definition
+func (b *Backend) GetDestination(destinationname string) (Destination, bool) {
+	for _, destination := range b.Destinations {
+		if destination.Name == destinationname {
+			return destination, true
+		}
+	}
+	return Destination{}, false
+}
+
+//GetRule getting a rule definition from the backend definition
+func (b *Backend) GetRule(rulename string) (Rule, bool) {
+	for _, rule := range b.Rules {
+		if rule.Name == rulename {
+			return rule, true
+		}
+	}
+	return Rule{}, false
+}
+
+//GetField getting a field definition from the model definition
+func (m *Model) GetField(fieldname string) (Field, bool) {
+	for _, field := range m.Fields {
+		if field.Name == fieldname {
+			return field, true
+		}
+	}
+	return Field{}, false
+}
+
+//GetIndex getting a index definition from the model definition
+func (m *Model) GetIndex(indexname string) (Index, bool) {
+	for _, index := range m.Indexes {
+		if index.Name == indexname {
+			return index, true
+		}
+	}
+	return Index{}, false
+}
+
+//GetFieldNames getting a list of fieldnames from the model definition
+func (m *Model) GetFieldNames() []string {
+	fields := make([]string, 0)
+	for _, field := range m.Fields {
+		fields = append(fields, field.Name)
+	}
+	return fields
+}

MessageService/model/task.go

@@ -3,7 +3,7 @@ package model
 import (
 	"encoding/json"
 
-	"github.com/willie68/AutoRestIoT/logging"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/logging"
 )
 
 const TaskOrphanedFilesReport = "orphanedFilesReport"

MessageService/test/postman/AutoRestIoT.postman_collection.json

@@ -0,0 +1,1259 @@
+{
+	"info": {
+		"_postman_id": "8521d47b-0fcf-44da-93a5-ce7d9ffc2081",
+		"name": "AutoRestIoT",
+		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
+	},
+	"item": [
+		{
+			"name": "DEV: get health",
+			"event": [
+				{
+					"listen": "test",
+					"script": {
+						"id": "00d61e0b-3b8d-4692-aeac-22f164cd1b1d",
+						"exec": [
+							"var responseJSON;",
+							"",
+							"try { ",
+							"    responseJSON = JSON.parse(responseBody); ",
+							"    tests['response is valid JSON'] = true;",
+							"}",
+							"catch (e) { ",
+							"    responseJSON = {}; ",
+							"    tests['response is valid JSON'] = false;",
+							"}",
+							"",
+							"",
+							"tests['response has post data'] = _.has(responseJSON, 'form');",
+							"tests['response matches the data posted'] = (responseJSON.form && responseJSON.form.strange === 'boom');"
+						],
+						"type": "text/javascript"
+					}
+				}
+			],
+			"protocolProfileBehavior": {
+				"disableBodyPruning": true
+			},
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "akteon00",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "w.klaas@gmx.de",
+							"type": "string"
+						},
+						{
+							"key": "saveHelperData",
+							"type": "any"
+						},
+						{
+							"key": "showPassword",
+							"value": false,
+							"type": "boolean"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [
+					{
+						"key": "Authorization",
+						"value": "Basic ZGV2dGVzdEBlYXN5LmRlOmVhc3lzcGlyaXQ="
+					},
+					{
+						"key": "Content-Type",
+						"value": "application/x-www-form-urlencoded"
+					},
+					{
+						"key": "X-mcs-apikey",
+						"value": "235778b0afd23afd1606f169a1b4d2ff"
+					},
+					{
+						"key": "Accept",
+						"value": "application/json",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "formdata",
+					"formdata": [
+						{
+							"key": "file",
+							"type": "file",
+							"src": "/E:/DATEN/easygit/GoBlobStore/test/postman/GoMicro.postman_collection.json"
+						}
+					]
+				},
+				"url": {
+					"raw": "https://127.0.0.1:9443/health/health",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"health",
+						"health"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl healthcheck",
+			"request": {
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "https://127.0.0.1:9443/health/health",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"health",
+						"health"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl readiness",
+			"request": {
+				"method": "GET",
+				"header": [],
+				"url": {
+					"raw": "https://127.0.0.1:9443/health/readiness",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"health",
+						"readiness"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl Get Users",
+			"protocolProfileBehavior": {
+				"disableBodyPruning": true
+			},
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "admin",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "admin",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [
+					{
+						"key": "X-mcs-apikey",
+						"value": "5854d123dd25f310395954f7c450171c",
+						"equals": true
+					},
+					{
+						"key": "X-mcs-system",
+						"value": "autorest-srv",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-tenant",
+						"value": "MCS",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-deleteref",
+						"value": "true",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"ApplicationName\": \"schematic\",\n    \"Description\": \"\",\n    \"Models\": [\n        {\n            \"Name\": \"schematic\",\n            \"Description\": \"\",\n            \"Fields\": [\n                {\n                    \"Name\": \"manufacturer\",\n                    \"Type\": \"string\",\n                    \"Mandantory\": true,\n                    \"Collection\": false\n                },\n                {\n                    \"Name\": \"model\",\n                    \"Type\": \"string\",\n                    \"Mandantory\": true,\n                    \"Collection\": false\n                },\n                {\n                    \"Name\": \"tags\",\n                    \"Type\": \"string\",\n                    \"Mandantory\": false,\n                    \"Collection\": true\n                }\n            ],\n            \"Indexes\": [\n                {\n                    \"Name\": \"fulltext\",\n                    \"Fields\": [\n                        \"manufactuer\",\n                        \"model\",\n                        \"tags\"\n                    ]\n                },\n                {\n                    \"Name\": \"manufacturer\",\n                    \"Fields\": [\n                        \"manufacturer\"\n                    ]\n                },\n                {\n                    \"Name\": \"tags\",\n                    \"Fields\": [\n                        \"tags\"\n                    ]\n                }\n            ]\n        }\n    ]\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "https://127.0.0.1:9443/api/v1/users/",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"api",
+						"v1",
+						"users",
+						""
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl changePWD",
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "admin",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "admin",
+							"type": "string"
+						}
+					]
+				},
+				"method": "PUT",
+				"header": [
+					{
+						"key": "X-mcs-apikey",
+						"value": "5854d123dd25f310395954f7c450171c",
+						"equals": true
+					},
+					{
+						"key": "X-mcs-system",
+						"value": "autorest-srv",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-tenant",
+						"value": "MCS",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-deleteref",
+						"value": "true",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "raw",
+					"raw": "{\r\n\t\"name\": \"guest\",\r\n\t\"password\": \"guest\",\r\n\t\"newpassword\": \"guest1\"\r\n}\r\n",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "https://127.0.0.1:9443/api/v1/users/guest",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"api",
+						"v1",
+						"users",
+						"guest"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl Post File",
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "editor",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "editor",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [
+					{
+						"key": "X-mcs-apikey",
+						"value": "5854d123dd25f310395954f7c450171c",
+						"equals": true
+					},
+					{
+						"key": "X-mcs-system",
+						"value": "autorest-srv",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-tenant",
+						"value": "MCS",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-deleteref",
+						"value": "true",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "formdata",
+					"formdata": [
+						{
+							"key": "file",
+							"type": "file",
+							"src": "/E:/DATEN/easygit/GoBlobStore/test/readme.md"
+						}
+					],
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "https://127.0.0.1:9443/api/v1/files/mybe/",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"api",
+						"v1",
+						"files",
+						"mybe",
+						""
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl Create Model",
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "editor",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "editor",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [
+					{
+						"key": "X-mcs-apikey",
+						"value": "5854d123dd25f310395954f7c450171c",
+						"equals": true
+					},
+					{
+						"key": "X-mcs-system",
+						"value": "autorest-srv",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-tenant",
+						"value": "MCS",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-deleteref",
+						"value": "true",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "raw",
+					"raw": "{\n    \"title\": \"title\",\n    \"foreignid\": \"12345678\",\n    \"date\": \"2020-04-09T09:46:06.252+02:00\",\n    \"array\": [\n        \"gut\",\n        \"besser\"\n    ],\n    \"file\": \"5e8ec42bd40bf595c1def721\",\n    \"number\": 1.234,\n    \"integer\": 2345,\n    \"bool_1\": 1,\n    \"bool_2\": true,\n    \"string\": \"string\",\n    \"map\": {\n        \"key_string\": \"value_string\",\n        \"key_int\": 2345,\n        \"key_num\": 2.345\n    }\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "https://127.0.0.1:9443/api/v1/models/mybe/mymodel/",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"api",
+						"v1",
+						"models",
+						"mybe",
+						"mymodel",
+						""
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl Get Model",
+			"protocolProfileBehavior": {
+				"disableBodyPruning": true
+			},
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "guest1",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "guest",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [
+					{
+						"key": "X-mcs-apikey",
+						"value": "5854d123dd25f310395954f7c450171c",
+						"equals": true
+					},
+					{
+						"key": "X-mcs-system",
+						"value": "autorest-srv",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-tenant",
+						"value": "MCS",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-deleteref",
+						"value": "true",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "raw",
+					"raw": "{\"ApplicationName\":\"schematic\",\"Description\":\"\",\"Models\":[{\"Name\":\"schematic\",\"Description\":\"\",\"Fields\":[{\"Name\":\"manufacturer\",\"Type\":\"string\",\"Mandantory\":true,\"Collection\":false},{\"Name\":\"model\",\"Type\":\"string\",\"Mandantory\":true,\"Collection\":false},{\"Name\":\"tags\",\"Type\":\"string\",\"Mandantory\":false,\"Collection\":true}],\"Indexes\":[{\"Name\":\"fulltext\",\"Fields\":[\"manufactuer\",\"model\",\"tags\"]},{\"Name\":\"manufacturer\",\"Fields\":[\"manufacturer\"]},{\"Name\":\"tags\",\"Fields\":[\"tags\"]}]}]}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "https://127.0.0.1:9443/api/v1/models/mybe/mymodel/5e8ec432d40bf595c1def723",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"api",
+						"v1",
+						"models",
+						"mybe",
+						"mymodel",
+						"5e8ec432d40bf595c1def723"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl Update Model",
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "editor",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "editor",
+							"type": "string"
+						}
+					]
+				},
+				"method": "PUT",
+				"header": [
+					{
+						"key": "X-mcs-apikey",
+						"value": "5854d123dd25f310395954f7c450171c",
+						"equals": true
+					},
+					{
+						"key": "X-mcs-system",
+						"value": "autorest-srv",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-tenant",
+						"value": "MCS",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-deleteref",
+						"value": "true",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "raw",
+					"raw": "{\r\n    \"ApplicationName\": \"schematic\",\r\n    \"Description\": \"\",\r\n    \"Models\": [\r\n        {\r\n            \"Description\": \"\",\r\n            \"Fields\": [\r\n                {\r\n                    \"Collection\": false,\r\n                    \"Mandantory\": true,\r\n                    \"Name\": \"manufacturer\",\r\n                    \"Type\": \"string\"\r\n                },\r\n                {\r\n                    \"Collection\": false,\r\n                    \"Mandantory\": true,\r\n                    \"Name\": \"model\",\r\n                    \"Type\": \"string\"\r\n                },\r\n                {\r\n                    \"Collection\": true,\r\n                    \"Mandantory\": false,\r\n                    \"Name\": \"tags\",\r\n                    \"Type\": \"string\"\r\n                }\r\n            ],\r\n            \"Indexes\": [\r\n                {\r\n                    \"Fields\": [\r\n                        \"manufactuer\",\r\n                        \"model\",\r\n                        \"tags\"\r\n                    ],\r\n                    \"Name\": \"fulltext\"\r\n                },\r\n                {\r\n                    \"Fields\": [\r\n                        \"manufacturer\"\r\n                    ],\r\n                    \"Name\": \"manufacturer\"\r\n                },\r\n                {\r\n                    \"Fields\": [\r\n                        \"tags\"\r\n                    ],\r\n                    \"Name\": \"tags\"\r\n                }\r\n            ],\r\n            \"Name\": \"schematic\"\r\n        }\r\n    ],\r\n    \"_created\": \"2020-04-07T09:13:24.606+02:00\",\r\n    \"_id\": \"5e8c2814867dfcf74776e418\",\r\n    \"_modified\": \"2020-04-07T09:13:24.606+02:00\",\r\n    \"_owner\": \"editor\"\r\n}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "https://127.0.0.1:9443/api/v1/models/mybe/mymodel/5e8c2814867dfcf74776e418",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"api",
+						"v1",
+						"models",
+						"mybe",
+						"mymodel",
+						"5e8c2814867dfcf74776e418"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl Delete Model",
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "editor",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "editor",
+							"type": "string"
+						}
+					]
+				},
+				"method": "DELETE",
+				"header": [
+					{
+						"key": "X-mcs-apikey",
+						"value": "5854d123dd25f310395954f7c450171c",
+						"equals": true
+					},
+					{
+						"key": "X-mcs-system",
+						"value": "autorest-srv",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-tenant",
+						"value": "MCS",
+						"type": "text"
+					}
+				],
+				"url": {
+					"raw": "https://127.0.0.1:9443/api/v1/models/schematics/schematic/5e8c8077451fc560b0d01f4c",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"api",
+						"v1",
+						"models",
+						"schematics",
+						"schematic",
+						"5e8c8077451fc560b0d01f4c"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl Delete Schematic Model",
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "editor",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "editor",
+							"type": "string"
+						}
+					]
+				},
+				"method": "DELETE",
+				"header": [
+					{
+						"key": "X-mcs-apikey",
+						"value": "5854d123dd25f310395954f7c450171c",
+						"equals": true
+					},
+					{
+						"key": "X-mcs-system",
+						"value": "autorest-srv",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-tenant",
+						"value": "MCS",
+						"type": "text"
+					}
+				],
+				"url": {
+					"raw": "https://127.0.0.1:9443/api/v1/models/schematics/schematic/5e8ec28b7043d7b6a7d0d1b5",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"api",
+						"v1",
+						"models",
+						"schematics",
+						"schematic",
+						"5e8ec28b7043d7b6a7d0d1b5"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl GET File",
+			"protocolProfileBehavior": {
+				"disableBodyPruning": true
+			},
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "editor",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "editor",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [
+					{
+						"key": "X-mcs-apikey",
+						"value": "5854d123dd25f310395954f7c450171c",
+						"equals": true
+					},
+					{
+						"key": "X-mcs-system",
+						"value": "autorest-srv",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-tenant",
+						"value": "MCS",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-deleteref",
+						"value": "true",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "formdata",
+					"formdata": [
+						{
+							"key": "file",
+							"type": "file",
+							"src": "/E:/DATEN/easygit/GoBlobStore/test/readme.md"
+						}
+					],
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "https://127.0.0.1:9443/api/v1/files/mybe/5e8b3d750666bc965feca1bb",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"api",
+						"v1",
+						"files",
+						"mybe",
+						"5e8b3d750666bc965feca1bb"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl Post backend",
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "editor",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "editor",
+							"type": "string"
+						}
+					]
+				},
+				"method": "POST",
+				"header": [
+					{
+						"key": "X-mcs-apikey",
+						"value": "5854d123dd25f310395954f7c450171c",
+						"equals": true
+					},
+					{
+						"key": "X-mcs-system",
+						"value": "autorest-srv",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-tenant",
+						"value": "MCS",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-deleteref",
+						"value": "true",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "formdata",
+					"formdata": [
+						{
+							"key": "file",
+							"type": "file",
+							"src": "/E:/DATEN/easygit/GoBlobStore/test/readme.md"
+						}
+					],
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "https://127.0.0.1:9443/api/v1/admin/backends/mybe/",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"api",
+						"v1",
+						"admin",
+						"backends",
+						"mybe",
+						""
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl Get backend names",
+			"protocolProfileBehavior": {
+				"disableBodyPruning": true
+			},
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "admin",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "admin",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [
+					{
+						"key": "X-mcs-apikey",
+						"value": "5854d123dd25f310395954f7c450171c",
+						"equals": true
+					},
+					{
+						"key": "X-mcs-system",
+						"value": "autorest-srv",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-tenant",
+						"value": "MCS",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-deleteref",
+						"value": "true",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "formdata",
+					"formdata": [
+						{
+							"key": "file",
+							"type": "file",
+							"src": "/E:/DATEN/easygit/GoBlobStore/test/readme.md"
+						}
+					],
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "https://127.0.0.1:9443/api/v1/admin/backends/",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"api",
+						"v1",
+						"admin",
+						"backends",
+						""
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl Delete backend",
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "admin",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "admin",
+							"type": "string"
+						}
+					]
+				},
+				"method": "DELETE",
+				"header": [
+					{
+						"key": "X-mcs-apikey",
+						"value": "5854d123dd25f310395954f7c450171c",
+						"equals": true
+					},
+					{
+						"key": "X-mcs-system",
+						"value": "autorest-srv",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-tenant",
+						"value": "MCS",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-deleteref",
+						"value": "true",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "formdata",
+					"formdata": [
+						{
+							"key": "file",
+							"type": "file",
+							"src": "/E:/DATEN/easygit/GoBlobStore/test/readme.md"
+						}
+					],
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "https://127.0.0.1:9443/api/v1/admin/backends/sensors/dropdata",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"api",
+						"v1",
+						"admin",
+						"backends",
+						"sensors",
+						"dropdata"
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "https://127.0.0.1:9443/api/v1/models/mybe/mymodel/",
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "guest",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "guest",
+							"type": "string"
+						},
+						{
+							"key": "saveHelperData",
+							"type": "any"
+						},
+						{
+							"key": "showPassword",
+							"value": false,
+							"type": "boolean"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [
+					{
+						"key": "X-mcs-apikey",
+						"value": "5854d123dd25f310395954f7c450171c",
+						"equals": true
+					},
+					{
+						"key": "X-mcs-system",
+						"value": "autorest-srv",
+						"type": "text"
+					},
+					{
+						"key": "X-mcs-tenant",
+						"value": "MCS",
+						"type": "text"
+					}
+				],
+				"url": {
+					"raw": "https://127.0.0.1:9443/api/v1/models/mybe/mymodel/",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"api",
+						"v1",
+						"models",
+						"mybe",
+						"mymodel",
+						""
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl Query Model",
+			"protocolProfileBehavior": {
+				"disableBodyPruning": true
+			},
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "editor",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "editor",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [
+					{
+						"equals": true,
+						"key": "X-mcs-apikey",
+						"value": "5854d123dd25f310395954f7c450171c"
+					},
+					{
+						"key": "X-mcs-system",
+						"type": "text",
+						"value": "autorest-srv"
+					},
+					{
+						"key": "X-mcs-tenant",
+						"type": "text",
+						"value": "MCS"
+					},
+					{
+						"key": "X-mcs-deleteref",
+						"type": "text",
+						"value": "true"
+					}
+				],
+				"body": {
+					"mode": "raw",
+					"raw": "{\"ApplicationName\":\"schematic\",\"Description\":\"\",\"Models\":[{\"Name\":\"schematic\",\"Description\":\"\",\"Fields\":[{\"Name\":\"manufacturer\",\"Type\":\"string\",\"Mandantory\":true,\"Collection\":false},{\"Name\":\"model\",\"Type\":\"string\",\"Mandantory\":true,\"Collection\":false},{\"Name\":\"tags\",\"Type\":\"string\",\"Mandantory\":false,\"Collection\":true}],\"Indexes\":[{\"Name\":\"fulltext\",\"Fields\":[\"manufactuer\",\"model\",\"tags\"]},{\"Name\":\"manufacturer\",\"Fields\":[\"manufacturer\"]},{\"Name\":\"tags\",\"Fields\":[\"tags\"]}]}]}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "https://127.0.0.1:9443/api/v1/models/mybe/mymodel/?query=",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"api",
+						"v1",
+						"models",
+						"mybe",
+						"mymodel",
+						""
+					],
+					"query": [
+						{
+							"key": "query",
+							"value": ""
+						}
+					]
+				}
+			},
+			"response": []
+		},
+		{
+			"name": "DEV: ssl Ruletest",
+			"protocolProfileBehavior": {
+				"disableBodyPruning": true
+			},
+			"request": {
+				"auth": {
+					"type": "basic",
+					"basic": [
+						{
+							"key": "password",
+							"value": "admin",
+							"type": "string"
+						},
+						{
+							"key": "username",
+							"value": "admin",
+							"type": "string"
+						}
+					]
+				},
+				"method": "GET",
+				"header": [
+					{
+						"equals": true,
+						"key": "X-mcs-apikey",
+						"value": "5854d123dd25f310395954f7c450171c"
+					},
+					{
+						"key": "X-mcs-system",
+						"type": "text",
+						"value": "autorest-srv"
+					},
+					{
+						"key": "X-mcs-tenant",
+						"type": "text",
+						"value": "MCS"
+					},
+					{
+						"key": "X-mcs-deleteref",
+						"type": "text",
+						"value": "true"
+					}
+				],
+				"body": {
+					"mode": "formdata",
+					"formdata": [
+						{
+							"key": "file",
+							"type": "file",
+							"src": "/E:/DATEN/easygit/GoBlobStore/test/readme.md"
+						}
+					],
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "https://127.0.0.1:9443/api/v1/admin/backends/sensors/",
+					"protocol": "https",
+					"host": [
+						"127",
+						"0",
+						"0",
+						"1"
+					],
+					"port": "9443",
+					"path": [
+						"api",
+						"v1",
+						"admin",
+						"backends",
+						"sensors",
+						""
+					]
+				}
+			},
+			"response": []
+		}
+	],
+	"protocolProfileBehavior": {}
+}

MessageService/worker/backend.go

@@ -0,0 +1,336 @@
+package worker
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"gopkg.in/yaml.v2"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/dao"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/model"
+)
+
+var BackendStorageRoute model.Route
+
+func init() {
+}
+
+//ValidateBackend validate if a backend definition is valid
+func ValidateBackend(be model.Backend) error {
+	// checking backendname format
+	// checking models
+	// checking indexes
+	// checking datasources
+	// checking rules
+	// checking destinations
+	return nil
+}
+
+//PrepareBackend will mmaily prepare the configuration of the datasources and destination to the right config type
+func PrepareBackend(backend model.Backend) (model.Backend, error) {
+	for i, dataSource := range backend.DataSources {
+		configJSON, err := json.Marshal(dataSource.Config)
+		switch dataSource.Type {
+		case "mqtt":
+			var config model.DataSourceConfigMQTT
+			if err = json.Unmarshal(configJSON, &config); err != nil {
+				return backend, errors.New(fmt.Sprintf("backend: %s, unmarshall mqtt config: %q", backend.Backendname, dataSource.Type))
+			}
+			backend.DataSources[i].Config = config
+		default:
+			return backend, errors.New(fmt.Sprintf("backend: %s, unknown datasource type: %q", backend.Backendname, dataSource.Type))
+		}
+	}
+	for i, destination := range backend.Destinations {
+		configJSON, err := json.Marshal(destination.Config)
+		switch destination.Type {
+		case "mqtt":
+			var config model.DataSourceConfigMQTT
+			if err = json.Unmarshal(configJSON, &config); err != nil {
+				return backend, errors.New(fmt.Sprintf("backend: %s, unmarshall mqtt config: %q", backend.Backendname, destination.Type))
+			}
+			backend.Destinations[i].Config = config
+		default:
+			return backend, errors.New(fmt.Sprintf("backend: %s, unknown destination type: %q", backend.Backendname, destination.Type))
+		}
+	}
+	return backend, nil
+}
+
+//RegisterBackend will create the needed indexes for the models and create the datasources, rules and destinations
+func RegisterBackend(backend model.Backend) error {
+	// create indexes if missing
+	models := backend.Models
+	for _, bemodel := range models {
+		err := createIndex(bemodel, backend.Backendname)
+		if err != nil {
+			log.Fatalf("%v", err)
+		}
+	}
+	// creating source plugins
+	for _, datasource := range backend.DataSources {
+		ok := false
+		for !ok {
+			err := createDatasource(datasource, backend.Backendname)
+			if err != nil {
+				log.Fatalf("%v", err)
+				time.Sleep(10 * time.Second)
+				continue
+			}
+			ok = true
+		}
+	}
+
+	for _, rule := range backend.Rules {
+		ok := false
+		for !ok {
+			err := createRule(rule, backend.Backendname)
+			if err != nil {
+				log.Fatalf("%v", err)
+				time.Sleep(10 * time.Second)
+				continue
+			}
+			ok = true
+		}
+	}
+
+	for _, destination := range backend.Destinations {
+		ok := false
+		for !ok {
+			err := Destinations.Register(backend.Backendname, destination)
+			if err != nil {
+				log.Fatalf("%v", err)
+				time.Sleep(10 * time.Second)
+				continue
+			}
+			ok = true
+		}
+	}
+
+	return nil
+}
+
+func createDatasource(datasource model.DataSource, backendname string) error {
+	switch datasource.Type {
+	case "mqtt":
+		clientID := fmt.Sprintf("autorestIoT.%s.%s", backendname, datasource.Name)
+		err := mqttRegisterTopic(clientID, backendname, datasource)
+		if err != nil {
+			return err
+		}
+	default:
+		log.Alertf("type \"%s\" is not availble as data source type", datasource.Type)
+	}
+	return nil
+}
+
+func destroyDatasource(datasource model.DataSource, backendname string) error {
+	switch datasource.Type {
+	case "mqtt":
+		clientID := fmt.Sprintf("autorestIoT.%s.%s", backendname, datasource.Name)
+		err := mqttDeregisterTopic(clientID, backendname, datasource)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func createRule(rule model.Rule, backendname string) error {
+	json, err := json.Marshal(rule.Transform)
+	if err != nil {
+		return err
+	}
+	err = Rules.Register(backendname, rule.Name, string(json))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func destroyRule(rule model.Rule, backendname string) error {
+	err := Rules.Deregister(backendname, rule.Name)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func createIndex(bemodel model.Model, backendname string) error {
+	indexes := bemodel.Indexes
+	// define stardard fulltext index
+	_, ok := bemodel.GetIndex(dao.FulltextIndexName)
+	if !ok {
+		fulltextIndex := model.Index{
+			Name:   dao.FulltextIndexName,
+			Fields: bemodel.GetFieldNames(),
+		}
+		indexes = append(indexes, fulltextIndex)
+	}
+	// define stardard indexes
+	for _, field := range bemodel.Fields {
+		_, ok := bemodel.GetIndex(dao.FulltextIndexName)
+		if !ok {
+			index := model.Index{
+				Name:   field.Name,
+				Fields: []string{field.Name},
+			}
+			indexes = append(indexes, index)
+		}
+	}
+	// Delete unused indexes
+	route := model.Route{
+		Backend: backendname,
+		Model:   bemodel.Name,
+	}
+	names, err := dao.GetStorage().GetIndexNames(route)
+	if err != nil {
+		return err
+	}
+	for _, idxName := range names {
+		found := false
+		for _, index := range indexes {
+			if idxName == index.Name {
+				found = true
+				break
+			}
+		}
+		if !found {
+			err = dao.GetStorage().DeleteIndex(route, idxName)
+		}
+	}
+	for _, index := range indexes {
+		err := dao.GetStorage().UpdateIndex(route, index)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+//DeregisterBackend will destroy all datasources, Rules and destinations and will remove the backend from the internal backendlist.
+func DeregisterBackend(backendname string) error {
+	backend, ok := model.BackendList.Get(backendname)
+	if ok {
+		for _, datasource := range backend.DataSources {
+			ok := false
+			for !ok {
+				err := destroyDatasource(datasource, backend.Backendname)
+				if err != nil {
+					log.Fatalf("%v", err)
+					return err
+				}
+				ok = true
+			}
+		}
+
+		for _, rule := range backend.Rules {
+			ok := false
+			for !ok {
+				err := destroyRule(rule, backend.Backendname)
+				if err != nil {
+					log.Fatalf("%v", err)
+					return err
+				}
+				ok = true
+			}
+		}
+
+		for _, destination := range backend.Destinations {
+			ok := false
+			for !ok {
+				err := Destinations.Deregister(backend.Backendname, destination)
+				if err != nil {
+					log.Fatalf("%v", err)
+					return err
+				}
+				ok = true
+			}
+		}
+
+		model.BackendList.Remove(backendname)
+	}
+	return nil
+}
+
+//StoreBackend will save the backend definition to the storage. If its already there, it will be updated
+func StoreBackend(backend model.Backend) (string, error) {
+	update := false
+	id := ""
+	query := fmt.Sprintf("{\"backendname\": \"%s\"}", backend.Backendname)
+	count, bemodels, err := dao.GetStorage().QueryModel(BackendStorageRoute, query, 0, 10)
+	if err != nil {
+		log.Alertf("%v", err)
+		return "", err
+	}
+	if count > 0 {
+		update = true
+		bemodel := model.JSONMap(bemodels[0])
+		id = bemodel["_id"].(primitive.ObjectID).Hex()
+		log.Infof("found backend with id: %s", id)
+	}
+	jsonString, err := json.Marshal(backend)
+	if err != nil {
+		return "", err
+	}
+
+	jsonModel := model.JSONMap{}
+	err = yaml.Unmarshal(jsonString, &jsonModel)
+	if err != nil {
+		return "", err
+	}
+	if update {
+		route := model.Route{
+			Backend:  BackendStorageRoute.Backend,
+			Apikey:   BackendStorageRoute.Apikey,
+			Identity: id,
+			Model:    BackendStorageRoute.Model,
+			SystemID: BackendStorageRoute.SystemID,
+			Username: BackendStorageRoute.Username,
+		}
+		_, err = dao.GetStorage().UpdateModel(route, jsonModel)
+		if err != nil {
+			return "", err
+		}
+		log.Infof("backend updated: %s", id)
+
+	} else {
+		id, err = dao.GetStorage().CreateModel(BackendStorageRoute, jsonModel)
+		if err != nil {
+			return "", err
+		}
+		log.Infof("backend created: %s", id)
+	}
+	return id, nil
+}
+
+//DeleteBackend deleting the backend from the storage, no data will be deleted
+func DeleteBackend(backendname string) error {
+	query := fmt.Sprintf("{\"backendname\": \"%s\"}", backendname)
+	count, bemodels, err := dao.GetStorage().QueryModel(BackendStorageRoute, query, 0, 10)
+	if err != nil {
+		log.Alertf("%v", err)
+		return err
+	}
+	if count > 0 {
+		bemodel := model.JSONMap(bemodels[0])
+		id := bemodel["_id"].(primitive.ObjectID).Hex()
+		log.Infof("found backend with id: %s", id)
+		route := model.Route{
+			Backend:  BackendStorageRoute.Backend,
+			Apikey:   BackendStorageRoute.Apikey,
+			Identity: id,
+			Model:    BackendStorageRoute.Model,
+			SystemID: BackendStorageRoute.SystemID,
+			Username: BackendStorageRoute.Username,
+		}
+		err = dao.GetStorage().DeleteModel(route)
+		if err != nil {
+			return err
+		}
+		log.Infof("backend deleted: %s", id)
+	}
+	return nil
+}

MessageService/worker/background.go

@@ -0,0 +1,190 @@
+package worker
+
+import (
+	"fmt"
+	"time"
+
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/dao"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/model"
+)
+
+const SystemBackend = "_system"
+const TaskModelName = "tasks"
+
+var lastChecked time.Time
+var backgroundConfig BackgroundConfig
+
+//BackgroundConfig configuration of background tasks
+type BackgroundConfig struct {
+	Period              int
+	DeleteOrphanedFiles bool
+}
+
+//InitBackgroundTasks initialise background tasks
+func InitBackgroundTasks(config BackgroundConfig) {
+	backgroundConfig = config
+	period := config.Period
+	log.Infof("healthcheck starting with period: %d seconds", period)
+	if period > 0 {
+		go func() {
+			background := time.NewTicker(time.Second * time.Duration(period))
+			for range background.C {
+				doTask()
+			}
+		}()
+	}
+}
+
+func GetTaskRoute() model.Route {
+	return model.Route{
+		Backend: SystemBackend,
+		Model:   TaskModelName,
+	}
+}
+
+//doTask internal function to process the background tasks
+func doTask() {
+	storage := dao.GetStorage()
+	// prepare the backend models, getting all models and backends with file fields
+	fileBackends := make([]map[string]string, 0)
+	for _, k := range model.BackendList.Names() {
+		backend, _ := model.BackendList.Get(k)
+		for _, m := range backend.Models {
+			for _, f := range m.Fields {
+				if f.Type == model.FieldTypeFile {
+					info := make(map[string]string)
+					info["backend"] = k
+					info["model"] = m.Name
+					info["field"] = f.Name
+					fileBackends = append(fileBackends, info)
+				}
+			}
+		}
+	}
+	storage.ProcessFiles(func(info model.FileInfo) bool {
+		if info.UploadDate.Add(1 * time.Hour).After(time.Now()) {
+			return false
+		}
+		toDelete := true
+		// log.Infof("found file: %s, id: %s, backend: %s", info.Filename, info.ID, info.Backend)
+		// get the right backend
+		for _, fileBackend := range fileBackends {
+			if info.Backend == fileBackend["backend"] {
+				route := model.Route{
+					Backend: info.Backend,
+					Model:   fileBackend["model"],
+				}
+				query := fmt.Sprintf("{ \"%s\": \"%s\"}", fileBackend["field"], info.ID)
+				count, _, _ := storage.QueryModel(route, query, 0, 0)
+				if count > 0 {
+					toDelete = false
+				}
+			}
+		}
+		log.Infof("file has to be deleted: %s", toDelete)
+		if toDelete && backgroundConfig.DeleteOrphanedFiles {
+			storage.DeleteFile(info.Backend, info.ID)
+		}
+		return toDelete
+	})
+	lastChecked = time.Now()
+}
+
+func reportOrphanedFiles() {
+	storage := dao.GetStorage()
+	taskRoute := GetTaskRoute()
+
+	task := model.Task{
+		Type:   model.TaskOrphanedFilesReport,
+		Status: model.New,
+	}
+	task, err := createTask(taskRoute, task)
+	if err != nil {
+		log.Alertf("error creating task: %v", err)
+		return
+	}
+
+	files := make([]model.FileInfo, 0)
+	// prepare the backend models, getting all models and backends with file fields
+	fileBackends := make([]map[string]string, 0)
+	for _, k := range model.BackendList.Names() {
+		backend, _ := model.BackendList.Get(k)
+		for _, m := range backend.Models {
+			for _, f := range m.Fields {
+				if f.Type == model.FieldTypeFile {
+					info := make(map[string]string)
+					info["backend"] = k
+					info["model"] = m.Name
+					info["field"] = f.Name
+					fileBackends = append(fileBackends, info)
+				}
+			}
+		}
+	}
+	task.Status = model.Running
+	updateTask(taskRoute, task)
+
+	err = storage.ProcessFiles(func(info model.FileInfo) bool {
+		if info.UploadDate.Add(1 * time.Hour).After(time.Now()) {
+			return false
+		}
+		toDelete := true
+		// log.Infof("found file: %s, id: %s, backend: %s", info.Filename, info.ID, info.Backend)
+		// get the right backend
+		for _, fileBackend := range fileBackends {
+			if info.Backend == fileBackend["backend"] {
+				route := model.Route{
+					Backend: info.Backend,
+					Model:   fileBackend["model"],
+				}
+				query := fmt.Sprintf("{ \"%s\": \"%s\"}", fileBackend["field"], info.ID)
+				count, _, _ := storage.QueryModel(route, query, 0, 0)
+				if count > 0 {
+					toDelete = false
+				}
+			}
+		}
+		log.Infof("file has to be deleted: %s", toDelete)
+		if toDelete {
+			files = append(files, info)
+		}
+		return toDelete
+	})
+	if err != nil {
+		log.Alertf("error: %s\n", err.Error())
+		return
+	}
+	task.Data = model.JSONMap{}
+	task.Data["fileids"] = files
+	task.Status = model.Finished
+	updateTask(taskRoute, task)
+
+}
+
+func updateTask(taskRoute model.Route, task model.Task) {
+	taskRoute.Identity = task.ID
+	jsonMap, err := task.ToJSONMap()
+	if err != nil {
+		return
+	}
+	_, err = dao.GetStorage().UpdateModel(taskRoute, jsonMap)
+	if err != nil {
+		log.Alertf("error updating task: %v", err)
+		return
+	}
+}
+
+func createTask(taskRoute model.Route, task model.Task) (model.Task, error) {
+	jsonMap, err := task.ToJSONMap()
+	if err != nil {
+		return model.Task{}, err
+	}
+	id, err := dao.GetStorage().CreateModel(taskRoute, jsonMap)
+	if err != nil {
+		log.Alertf("error creating task: %v", err)
+		return model.Task{}, err
+	}
+
+	task.ID = id
+	return task, nil
+}

MessageService/worker/destinations.go

@@ -0,0 +1,115 @@
+package worker
+
+import (
+	"errors"
+	"fmt"
+
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/model"
+)
+
+//DestinationProcessor fpr every destination there must be a processor to do the work
+type DestinationProcessor interface {
+	//Initialise this procssor
+	Initialise(backend string, destination model.Destination) error
+	//Destroy this processor
+	Destroy(backend string, destination model.Destination) error
+	//Store do the right storage
+	Store(data model.JSONMap) (string, error)
+}
+
+//NullDestinationProcessor does nothing
+type NullDestinationProcessor struct {
+}
+
+//Initialise do nothing on initialise
+func (n *NullDestinationProcessor) Initialise(backend string, destination model.Destination) error {
+	return nil
+}
+
+//Destroy do nothing on initialise
+func (n *NullDestinationProcessor) Destroy(backend string, destination model.Destination) error {
+	return nil
+}
+
+//Store do nothing on store
+func (n *NullDestinationProcessor) Store(data model.JSONMap) (string, error) {
+	return "noId", nil
+}
+
+//ErrDestinationNotFound the destination was not found in this system
+var ErrDestinationNotFound = errors.New("Missing destination")
+
+//DestinationList list type
+type DestinationList struct {
+	destinations map[string]model.Destination
+	processors   map[string]DestinationProcessor
+}
+
+func GetNewDestinationProcessor(backend string, destination model.Destination) (DestinationProcessor, error) {
+	switch destination.Type {
+	case "mqtt":
+		return CreateMQTTDestinationProcessor(backend, destination)
+	case "null":
+		return &NullDestinationProcessor{}, nil
+	default:
+		return &NullDestinationProcessor{}, nil
+	}
+}
+
+//Destinations List off all registered destinations
+var Destinations = DestinationList{
+	destinations: make(map[string]model.Destination),
+}
+
+//Register registering a new destination under the right name
+func (d *DestinationList) Register(backendName string, destination model.Destination) error {
+	destinationNsName := GetDestinationNsName(backendName, destination.Name)
+	d.destinations[destinationNsName] = destination
+	return nil
+}
+
+//Deregister deregistering a new destination with a name
+func (d *DestinationList) Deregister(backendName string, destination model.Destination) error {
+	destinationNsName := GetDestinationNsName(backendName, destination.Name)
+	// getting the processor for this
+	processor, ok := d.processors[destinationNsName]
+	if ok {
+		err := processor.Destroy(backendName, destination)
+		if err != nil {
+			return err
+		}
+		delete(d.processors, destinationNsName)
+	}
+	// removing the destination from the list
+	delete(d.destinations, destinationNsName)
+	return nil
+}
+
+//Store storing a message into the desired destination
+func (d *DestinationList) Store(backendName string, destinationName string, data model.JSONMap) error {
+	destinationNsName := GetDestinationNsName(backendName, destinationName)
+	destination, ok := d.destinations[destinationNsName]
+	if !ok {
+		return ErrDestinationNotFound
+	}
+	var processor DestinationProcessor
+	processor, ok = d.processors[destinationName]
+	if !ok {
+		var err error
+		processor, err = GetNewDestinationProcessor(backendName, destination)
+		if err != nil {
+			return err
+		}
+	}
+	_, err := processor.Store(data)
+	if err != nil {
+		return err
+	}
+	//log.Infof("store object in destination %s as %s", destination, id)
+	return nil
+}
+
+//GetDestinationNsName getting the unique name of a backend destination
+func GetDestinationNsName(backendName, destinationName string) string {
+	return fmt.Sprintf("%s.%s", backendName, destinationName)
+}

MessageService/worker/model.go

@@ -12,11 +12,11 @@ import (
 	"sync"
 	"time"
 
-	"github.com/willie68/AutoRestIoT/config"
-	"github.com/willie68/AutoRestIoT/dao"
-	"github.com/willie68/AutoRestIoT/internal"
-	"github.com/willie68/AutoRestIoT/logging"
-	"github.com/willie68/AutoRestIoT/model"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/config"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/dao"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/internal"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/logging"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/model"
 )
 
 //ErrMissingID the id of the model is mandatory and not availble

MessageService/worker/mqtt.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	mqtt "github.com/eclipse/paho.mqtt.golang"
-	"github.com/willie68/AutoRestIoT/model"
+	"wkla.no-ip.biz/gogs/Willie/MsgService/MessageService/model"
 )
 
 const processorPrefix = "processor"