WilliesSchematicsWorld/schematic-service-go/dao/mongodao.go

package dao

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"strings"
	"time"

	"github.com/willie68/schematic-service-go/config"
	slicesutils "github.com/willie68/schematic-service-go/internal"
	"github.com/willie68/schematic-service-go/model"

	"go.mongodb.org/mongo-driver/bson"
	"go.mongodb.org/mongo-driver/bson/primitive"
	"go.mongodb.org/mongo-driver/mongo"
	"go.mongodb.org/mongo-driver/mongo/gridfs"
	"go.mongodb.org/mongo-driver/mongo/options"
)

// time to reload all users
const userReloadPeriod = 1 * time.Hour
const timeout = 1 * time.Minute
const attachmentsCollectionName = "attachments"
const schematicsCollectionName = "schematics"
const tagsCollectionName = "tags"
const manufacturersCollectionName = "manufacturers"
const usersCollectionName = "users"
const effectsCollectionName = "effects"
const effectTypesCollectionName = "effectTypes"

// MongoDAO a mongodb based dao
type MongoDAO struct {
	initialised   bool
	client        *mongo.Client
	mongoConfig   config.MongoDB
	bucket        gridfs.Bucket
	database      mongo.Database
	tags          model.Tags
	manufacturers model.Manufacturers
	users         map[string]string
	ticker        time.Ticker
	done          chan bool
}

// InitDAO initialise the mongodb connection, build up all collections and indexes
func (m *MongoDAO) InitDAO(MongoConfig config.MongoDB) {
	m.initialised = false
	m.mongoConfig = MongoConfig
	//	uri := fmt.Sprintf("mongodb://%s:%s@%s:%d", mongoConfig.Username, mongoConfig.Password, mongoConfig.Host, mongoConfig.Port)
	uri := fmt.Sprintf("mongodb://%s:%d", m.mongoConfig.Host, m.mongoConfig.Port)
	clientOptions := options.Client()
	clientOptions.ApplyURI(uri)
	clientOptions.Auth = &options.Credential{Username: m.mongoConfig.Username, Password: m.mongoConfig.Password, AuthSource: m.mongoConfig.AuthDB}
	var err error
	m.client, err = mongo.NewClient(clientOptions)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
	}
	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()
	err = m.client.Connect(ctx)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
	}
	m.database = *m.client.Database(m.mongoConfig.Database)

	myBucket, err := gridfs.NewBucket(&m.database, options.GridFSBucket().SetName(attachmentsCollectionName))
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
	}
	m.bucket = *myBucket

	m.initIndexSchematics()
	m.initIndexTags()
	m.initIndexManufacturers()
	m.initIndexEffectTypes()
	m.initIndexEffects()

	m.tags = model.NewTags()
	m.manufacturers = model.NewManufacturers()
	m.users = make(map[string]string)
	m.initTags()
	m.initManufacturers()
	m.initUsers()
	m.initialised = true
}

func (m *MongoDAO) initIndexSchematics() {
	collection := m.database.Collection(schematicsCollectionName)
	indexView := collection.Indexes()
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	cursor, err := indexView.List(ctx)
	if err != nil {
		log.Fatal(err)
	}
	defer cursor.Close(ctx)
	myIndexes := make([]string, 0)
	for cursor.Next(ctx) {
		var index bson.M
		if err = cursor.Decode(&index); err != nil {
			log.Fatal(err)
		}
		myIndexes = append(myIndexes, index["name"].(string))
	}

	if !slicesutils.Contains(myIndexes, "manufacturer") {
		ctx, _ = context.WithTimeout(context.Background(), timeout)
		models := []mongo.IndexModel{
			{
				Keys:    bson.D{{"manufacturer", 1}},
				Options: options.Index().SetName("manufacturer").SetCollation(&options.Collation{Locale: "en", Strength: 2}),
			},
			{
				Keys:    bson.D{{"model", 1}},
				Options: options.Index().SetName("model").SetCollation(&options.Collation{Locale: "en", Strength: 2}),
			},
			{
				Keys:    bson.D{{"tags", 1}},
				Options: options.Index().SetName("tags").SetCollation(&options.Collation{Locale: "en", Strength: 2}),
			},
			{
				Keys:    bson.D{{"subtitle", 1}},
				Options: options.Index().SetName("subtitle").SetCollation(&options.Collation{Locale: "en", Strength: 2}),
			},
			{
				Keys:    bson.D{{"manufacturer", "text"}, {"model", "text"}, {"tags", "text"}, {"subtitle", "text"}, {"description", "text"}, {"owner", "text"}},
				Options: options.Index().SetName("$text"),
			},
		}

		// Specify the MaxTime option to limit the amount of time the operation can run on the server
		opts := options.CreateIndexes().SetMaxTime(2 * time.Second)
		names, err := indexView.CreateMany(context.TODO(), models, opts)
		if err != nil {
			log.Fatal(err)
		}
		log.Print("create indexes:")
		for _, name := range names {
			log.Println(name)
		}
	}
}

func (m *MongoDAO) initIndexTags() {
	collection := m.database.Collection(tagsCollectionName)
	indexView := collection.Indexes()
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	cursor, err := indexView.List(ctx)
	if err != nil {
		log.Fatal(err)
	}
	defer cursor.Close(ctx)
	myIndexes := make([]string, 0)
	for cursor.Next(ctx) {
		var index bson.M
		if err = cursor.Decode(&index); err != nil {
			log.Fatal(err)
		}
		myIndexes = append(myIndexes, index["name"].(string))
	}

	if !slicesutils.Contains(myIndexes, "name") {
		ctx, _ = context.WithTimeout(context.Background(), timeout)
		models := []mongo.IndexModel{
			{
				Keys:    bson.D{{"name", 1}},
				Options: options.Index().SetUnique(true).SetName("name").SetCollation(&options.Collation{Locale: "en", Strength: 2}),
			},
		}

		// Specify the MaxTime option to limit the amount of time the operation can run on the server
		opts := options.CreateIndexes().SetMaxTime(2 * time.Second)
		names, err := indexView.CreateMany(context.TODO(), models, opts)
		if err != nil {
			log.Fatal(err)
		}
		log.Print("create indexes:")
		for _, name := range names {
			log.Println(name)
		}
	}
}

func (m *MongoDAO) initIndexManufacturers() {
	collection := m.database.Collection(manufacturersCollectionName)
	indexView := collection.Indexes()
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	cursor, err := indexView.List(ctx)
	if err != nil {
		log.Fatal(err)
	}
	defer cursor.Close(ctx)
	myIndexes := make([]string, 0)
	for cursor.Next(ctx) {
		var index bson.M
		if err = cursor.Decode(&index); err != nil {
			log.Fatal(err)
		}
		myIndexes = append(myIndexes, index["name"].(string))
	}

	if !slicesutils.Contains(myIndexes, "name") {
		ctx, _ = context.WithTimeout(context.Background(), timeout)
		models := []mongo.IndexModel{
			{
				Keys:    bson.D{{"name", 1}},
				Options: options.Index().SetUnique(true).SetName("name").SetCollation(&options.Collation{Locale: "en", Strength: 2}),
			},
		}

		// Specify the MaxTime option to limit the amount of time the operation can run on the server
		opts := options.CreateIndexes().SetMaxTime(2 * time.Second)
		names, err := indexView.CreateMany(context.TODO(), models, opts)
		if err != nil {
			log.Fatal(err)
		}
		log.Print("create indexes:")
		for _, name := range names {
			log.Println(name)
		}
	}
}

func (m *MongoDAO) initIndexEffects() {
	collection := m.database.Collection(effectsCollectionName)
	indexView := collection.Indexes()
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	cursor, err := indexView.List(ctx)
	if err != nil {
		log.Fatal(err)
	}
	defer cursor.Close(ctx)
	myIndexes := make([]string, 0)
	for cursor.Next(ctx) {
		var index bson.M
		if err = cursor.Decode(&index); err != nil {
			log.Fatal(err)
		}
		myIndexes = append(myIndexes, index["name"].(string))
	}
	if !slicesutils.Contains(myIndexes, "effectType") {
		ctx, _ = context.WithTimeout(context.Background(), timeout)
		models := []mongo.IndexModel{
			{
				Keys:    bson.D{{"effectType", 1}},
				Options: options.Index().SetName("effectType").SetCollation(&options.Collation{Locale: "en", Strength: 2}),
			},
			{
				Keys:    bson.D{{"manufacturer", 1}},
				Options: options.Index().SetName("manufacturer").SetCollation(&options.Collation{Locale: "en", Strength: 2}),
			},
			{
				Keys:    bson.D{{"model", 1}},
				Options: options.Index().SetName("model").SetCollation(&options.Collation{Locale: "en", Strength: 2}),
			},
			{
				Keys:    bson.D{{"tags", 1}},
				Options: options.Index().SetName("tags").SetCollation(&options.Collation{Locale: "en", Strength: 2}),
			},
			{
				Keys:    bson.D{{"comment", 1}},
				Options: options.Index().SetName("comment").SetCollation(&options.Collation{Locale: "en", Strength: 2}),
			},
			{
				Keys:    bson.D{{"effectType", "text"}, {"manufacturer", "text"}, {"model", "text"}, {"tags", "text"}, {"comment", "text"}, {"connector", "text"}, {"current", "text"}, {"voltage", "text"}},
				Options: options.Index().SetName("$text"),
			},
		}

		// Specify the MaxTime option to limit the amount of time the operation can run on the server
		opts := options.CreateIndexes().SetMaxTime(2 * time.Second)
		names, err := indexView.CreateMany(context.TODO(), models, opts)
		if err != nil {
			log.Fatal(err)
		}
		log.Print("create indexes:")
		for _, name := range names {
			log.Println(name)
		}
	}
}

func (m *MongoDAO) initIndexEffectTypes() {
	collection := m.database.Collection(effectTypesCollectionName)
	indexView := collection.Indexes()
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	cursor, err := indexView.List(ctx)
	if err != nil {
		log.Fatal(err)
	}
	defer cursor.Close(ctx)
	myIndexes := make([]string, 0)
	for cursor.Next(ctx) {
		var index bson.M
		if err = cursor.Decode(&index); err != nil {
			log.Fatal(err)
		}
		myIndexes = append(myIndexes, index["name"].(string))
	}
	if !slicesutils.Contains(myIndexes, "typeName") {
		ctx, _ = context.WithTimeout(context.Background(), timeout)
		models := []mongo.IndexModel{
			{
				Keys:    bson.D{{"typeName", 1}},
				Options: options.Index().SetName("typeName").SetCollation(&options.Collation{Locale: "en", Strength: 2}),
			},
		}

		// Specify the MaxTime option to limit the amount of time the operation can run on the server
		opts := options.CreateIndexes().SetMaxTime(2 * time.Second)
		names, err := indexView.CreateMany(context.TODO(), models, opts)
		if err != nil {
			log.Fatal(err)
		}
		log.Print("create indexes:")
		for _, name := range names {
			log.Println(name)
		}
	}
}

func (m *MongoDAO) initTags() {
	m.reloadTags()
}

func (m *MongoDAO) initManufacturers() {
	m.reloadManufacturers()
}

func (m *MongoDAO) initUsers() {
	m.reloadUsers()

	go func() {
		background := time.NewTicker(userReloadPeriod)
		for _ = range background.C {
			m.reloadUsers()
			m.reloadTags()
			m.reloadManufacturers()
		}
	}()
}

func (m *MongoDAO) reloadUsers() {
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	usersCollection := m.database.Collection(usersCollectionName)
	cursor, err := usersCollection.Find(ctx, bson.M{})
	if err != nil {
		log.Fatal(err)
	}
	defer cursor.Close(ctx)
	localUsers := make(map[string]string)
	for cursor.Next(ctx) {
		var user bson.M
		if err = cursor.Decode(&user); err != nil {
			log.Fatal(err)
		} else {
			username := strings.ToLower(user["name"].(string))
			password := user["password"].(string)
			localUsers[username] = BuildPasswordHash(password)
		}
	}
	m.users = localUsers
	if len(m.users) == 0 {
		admin := model.User{
			Name:     "w.klaas@gmx.de",
			Password: "akteon0000",
			Admin:    true,
		}
		m.AddUser(admin)
		guest := model.User{
			Name:     "gast",
			Password: "gast1234",
			Admin:    false,
			Guest:    true,
		}
		m.AddUser(guest)
	}
}

func (m *MongoDAO) reloadTags() {
	myTags := model.NewTags()

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	tagsCollection := m.database.Collection(tagsCollectionName)
	cursor, err := tagsCollection.Find(ctx, bson.M{})
	if err != nil {
		log.Fatal(err)
	}
	defer cursor.Close(ctx)
	tagList := make([]string, 0)
	for cursor.Next(ctx) {
		var tag bson.M
		if err = cursor.Decode(&tag); err != nil {
			log.Fatal(err)
		} else {
			tagList = append(tagList, tag["name"].(string))
		}
	}

	ctx, _ = context.WithTimeout(context.Background(), timeout)
	schematicCollection := m.database.Collection(schematicsCollectionName)

	for _, tagName := range tagList {
		queryDoc := bson.M{
			"tags": tagName,
		}
		n, err := schematicCollection.CountDocuments(ctx, queryDoc, &options.CountOptions{Collation: &options.Collation{Locale: "en", Strength: 2}})
		if err != nil {
			log.Println(err)
		}
		//log.Printf("tag: %s has %d schematics.", tagName, n)
		myTags.Add(tagName, int(n))
	}
	m.tags = myTags
}

func (m *MongoDAO) reloadManufacturers() {
	myManufacturer := model.NewManufacturers()

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	manufacturersCollection := m.database.Collection(manufacturersCollectionName)
	cursor, err := manufacturersCollection.Find(ctx, bson.M{})
	if err != nil {
		log.Fatal(err)
	}
	defer cursor.Close(ctx)
	manuList := make([]string, 0)
	for cursor.Next(ctx) {
		var manufacturer bson.M
		if err = cursor.Decode(&manufacturer); err != nil {
			log.Fatal(err)
		} else {
			manuList = append(manuList, manufacturer["name"].(string))
		}
	}

	ctx, _ = context.WithTimeout(context.Background(), timeout)
	schematicCollection := m.database.Collection(schematicsCollectionName)

	for _, manuName := range manuList {
		queryDoc := bson.M{
			"manufacturer": manuName,
		}
		n, err := schematicCollection.CountDocuments(ctx, queryDoc, &options.CountOptions{Collation: &options.Collation{Locale: "en", Strength: 2}})
		if err != nil {
			log.Println(err)
		}
		//log.Printf("manufacturer: %s has %d schematics.", manuName, n)
		myManufacturer.Add(manuName, int(n))
	}
	m.manufacturers = myManufacturer
}

// AddFile adding a file to the storage, stream like
func (m *MongoDAO) AddFile(filename string, reader io.Reader) (string, error) {
	uploadOpts := options.GridFSUpload().SetMetadata(bson.D{{"tag", "tag"}})

	fileID, err := m.bucket.UploadFromStream(filename, reader, uploadOpts)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
		return "", err
	}
	log.Printf("Write file to DB was successful. File id: %s \n", fileID)
	id := fileID.Hex()
	return id, nil
}

// CreateSchematic creating a new schematic in the database
func (m *MongoDAO) CreateSchematic(schematic model.Schematic) (string, error) {

	for _, tag := range schematic.Tags {
		if !m.tags.Contains(tag) {
			m.CreateTag(tag)
		}
	}

	if !m.manufacturers.Contains(schematic.Manufacturer) {
		m.CreateManufacturer(schematic.Manufacturer)
	}

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(schematicsCollectionName)
	result, err := collection.InsertOne(ctx, schematic)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
		return "", err
	}
	filter := bson.M{"_id": result.InsertedID}
	err = collection.FindOne(ctx, filter).Decode(&schematic)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
		return "", err
	}
	switch v := result.InsertedID.(type) {
	case primitive.ObjectID:
		return v.Hex(), nil
	}
	return "", nil
}

// UpdateSchematic creating a new schematic in the database
func (m *MongoDAO) UpdateSchematic(schematic model.Schematic) (string, error) {

	for _, tag := range schematic.Tags {
		if !m.tags.Contains(tag) {
			m.CreateTag(tag)
		}
	}

	if !m.manufacturers.Contains(schematic.Manufacturer) {
		m.CreateManufacturer(schematic.Manufacturer)
	}

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(schematicsCollectionName)
	filter := bson.M{"_id": schematic.ID}
	updateDoc := bson.D{{"$set", schematic}}
	result, err := collection.UpdateOne(ctx, filter, updateDoc)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
		return "", err
	}
	if result.ModifiedCount != 1 {
		return "", errors.New("can't update document.")
	}
	err = collection.FindOne(ctx, filter).Decode(&schematic)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
		return "", err
	}
	return schematic.ID.Hex(), nil
}

// GetSchematic getting a sdingle schematic
func (m *MongoDAO) GetSchematic(schematicID string) (model.Schematic, error) {
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	schematicCollection := m.database.Collection(schematicsCollectionName)
	objectID, _ := primitive.ObjectIDFromHex(schematicID)
	result := schematicCollection.FindOne(ctx, bson.M{"_id": objectID})
	err := result.Err()
	if err == mongo.ErrNoDocuments {
		log.Print(err)
		return model.Schematic{}, ErrNoDocument
	}
	if err != nil {
		log.Print(err)
		return model.Schematic{}, err
	}
	var schematic model.Schematic
	if err := result.Decode(&schematic); err != nil {
		log.Print(err)
		return model.Schematic{}, err
	} else {
		return schematic, nil
	}
}

// DeleteSchematic getting a sdingle schematic
func (m *MongoDAO) DeleteSchematic(schematicID string) error {
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	schematicCollection := m.database.Collection(schematicsCollectionName)
	objectID, _ := primitive.ObjectIDFromHex(schematicID)
	result, err := schematicCollection.DeleteOne(ctx, bson.M{"_id": objectID})
	if err != nil {
		log.Print(err)
		return err
	} else {
		if result.DeletedCount > 0 {
			return nil
		}
		return ErrNoDocument
	}
}

//GetFile getting a single from the database with the id
func (m *MongoDAO) GetFilename(fileid string) (string, error) {
	objectID, err := primitive.ObjectIDFromHex(fileid)
	if err != nil {
		log.Print(err)
		return "", err
	}
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	cursor, err := m.bucket.Find(bson.M{"_id": objectID})
	if err != nil {
		log.Print(err)
		return "", err
	}
	defer cursor.Close(ctx)
	cursor.Next(ctx)
	var file bson.M
	var filename string
	if err = cursor.Decode(&file); err != nil {
		log.Print(err)
		return "", err
	} else {
		filename = file["filename"].(string)
	}
	return filename, nil
}

//GetFile getting a single from the database with the id
func (m *MongoDAO) GetFile(fileid string, stream io.Writer) error {
	objectID, err := primitive.ObjectIDFromHex(fileid)
	if err != nil {
		log.Print(err)
		return err
	}
	_, err = m.bucket.DownloadToStream(objectID, stream)
	if err != nil {
		log.Print(err)
		return err
	}
	return nil
}

// GetSchematics getting a sdingle schematic
func (m *MongoDAO) GetSchematics(query string, offset int, limit int, owner string) (int64, []model.Schematic, error) {
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	schematicCollection := m.database.Collection(schematicsCollectionName)
	var queryM map[string]interface{}
	err := json.Unmarshal([]byte(query), &queryM)
	if err != nil {
		log.Print(err)
		return 0, nil, err
	}
	queryDoc := bson.M{}
	for k, v := range queryM {
		if k == "$fulltext" {
			queryDoc["$text"] = bson.M{"$search": v}
		} else {
			switch v := v.(type) {
			//			case float64:
			//			case int:
			//			case bool:
			case string:
				queryDoc[k] = bson.M{"$regex": v}
			}
			//queryDoc[k] = v
		}
	}
	data, _ := json.Marshal(queryDoc)
	log.Printf("mongoquery: %s\n", string(data))
	n, err := schematicCollection.CountDocuments(ctx, queryDoc, &options.CountOptions{Collation: &options.Collation{Locale: "en", Strength: 2}})
	if err != nil {
		log.Print(err)
		return 0, nil, err
	}
	cursor, err := schematicCollection.Find(ctx, queryDoc, &options.FindOptions{Collation: &options.Collation{Locale: "en", Strength: 2}})
	if err != nil {
		log.Print(err)
		return 0, nil, err
	}
	defer cursor.Close(ctx)
	schematics := make([]model.Schematic, 0)
	count := 0
	docs := 0
	for cursor.Next(ctx) {
		if count >= offset {
			if docs < limit {
				var schematic model.Schematic
				if err = cursor.Decode(&schematic); err != nil {
					log.Print(err)
					return 0, nil, err
				} else {
					if !schematic.PrivateFile || schematic.Owner == owner {
						schematics = append(schematics, schematic)
						docs++
					}
				}
			} else {
				break
			}
		}
		count++
	}
	return n, schematics, nil
}

// GetSchematicsCount getting a sdingle schematic
func (m *MongoDAO) GetSchematicsCount(query string, owner string) (int64, error) {
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	schematicCollection := m.database.Collection(schematicsCollectionName)
	queryDoc := bson.M{}

	if query != "" {
		var queryM map[string]interface{}
		err := json.Unmarshal([]byte(query), &queryM)
		if err != nil {
			log.Print(err)
			return 0, err
		}
		for k, v := range queryM {
			if k == "$fulltext" {
				queryDoc["$text"] = bson.M{"$search": v}
			} else {
				switch v := v.(type) {
				//			case float64:
				//			case int:
				//			case bool:
				case string:
					queryDoc[k] = bson.M{"$regex": v}
				}
				//queryDoc[k] = v
			}
		}
		data, _ := json.Marshal(queryDoc)
		log.Printf("mongoquery: %s\n", string(data))
	}
	n, err := schematicCollection.CountDocuments(ctx, queryDoc, &options.CountOptions{Collation: &options.Collation{Locale: "en", Strength: 2}})
	if err != nil {
		log.Print(err)
		return 0, err
	}
	return n, nil
}

// CreateTag create a new tag in the storage
func (m *MongoDAO) CreateTag(tag string) error {
	tag = strings.ToLower(tag)
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(tagsCollectionName)
	tagModel := bson.M{"name": tag}
	_, err := collection.InsertOne(ctx, tagModel)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
		return err
	}
	m.tags.Add(tag, 1)
	return nil
}

//GetTags getting a list of all tags
func (m *MongoDAO) GetTags() []model.Tag {
	sort.Slice(m.tags.List, func(i, j int) bool { return m.tags.List[i].Name < m.tags.List[j].Name })
	return m.tags.List
}

// GetTagsCount getting the count of all tags
func (m *MongoDAO) GetTagsCount() int {
	return len(m.tags.List)
}

// CreateManufacturer create a new manufacturer in the storage
func (m *MongoDAO) CreateManufacturer(manufacturer string) error {
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(manufacturersCollectionName)
	manufacturerModel := bson.M{"name": manufacturer}
	_, err := collection.InsertOne(ctx, manufacturerModel)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
		return err
	}
	m.manufacturers.Add(manufacturer, 1)
	return nil
}

// GetManufacturers getting a list of all manufacturers
func (m *MongoDAO) GetManufacturers() []model.Manufacturer {
	sort.Slice(m.manufacturers.List, func(i, j int) bool { return m.manufacturers.List[i].Name < m.manufacturers.List[j].Name })
	return m.manufacturers.List
}

// GetManufacturersCount getting the count of all manufacturers
func (m *MongoDAO) GetManufacturersCount() int {
	return len(m.manufacturers.List)
}

//CreateEffectType cerating a new effect type
func (m *MongoDAO) CreateEffectType(effectType model.EffectType) (string, error) {
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(effectTypesCollectionName)
	result, err := collection.InsertOne(ctx, effectType)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
		return "", err
	}
	filter := bson.M{"_id": result.InsertedID}
	err = collection.FindOne(ctx, filter).Decode(&effectType)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
		return "", err
	}
	switch v := result.InsertedID.(type) {
	case primitive.ObjectID:
		return v.Hex(), nil
	}
	return "", nil
}

// GetEffectTypes getting a sdingle schematic
func (m *MongoDAO) GetEffectTypes() ([]model.EffectType, error) {
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(effectTypesCollectionName)
	findOptions := options.Find()
	// Sort by `price` field descending
	findOptions.SetSort(bson.D{{"typeName", 1}})
	queryDoc := bson.M{}
	cursor, err := collection.Find(ctx, queryDoc, &options.FindOptions{Collation: &options.Collation{Locale: "en", Strength: 2}, Sort: bson.D{{"typeName", 1}}})
	if err != nil {
		log.Print(err)
		return nil, err
	}
	defer cursor.Close(ctx)
	effectTypes := make([]model.EffectType, 0)
	for cursor.Next(ctx) {
		var effectType model.EffectType
		if err = cursor.Decode(&effectType); err != nil {
			log.Print(err)
			return nil, err
		} else {
			effectTypes = append(effectTypes, effectType)
		}
	}
	return effectTypes, nil
}

//CreateEffect cerating a new effect type
func (m *MongoDAO) CreateEffect(effect model.Effect) (string, error) {

	for _, tag := range effect.Tags {
		if !m.tags.Contains(tag) {
			m.CreateTag(tag)
		}
	}

	if !m.manufacturers.Contains(effect.Manufacturer) {
		m.CreateManufacturer(effect.Manufacturer)
	}

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(effectsCollectionName)
	result, err := collection.InsertOne(ctx, effect)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
		return "", err
	}
	filter := bson.M{"_id": result.InsertedID}
	err = collection.FindOne(ctx, filter).Decode(&effect)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
		return "", err
	}
	switch v := result.InsertedID.(type) {
	case primitive.ObjectID:
		return v.Hex(), nil
	}
	return "", nil
}

//GetEffectsCount getting the count  of effects regarding to the query
func (m *MongoDAO) GetEffectsCount(query string) (int, error) {
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(effectsCollectionName)
	queryDoc := bson.M{}
	if query != "" {
		var queryM map[string]interface{}
		err := json.Unmarshal([]byte(query), &queryM)
		if err != nil {
			log.Print(err)
			return 0, err
		}
		for k, v := range queryM {
			if k == "$fulltext" {
				queryDoc["$text"] = bson.M{"$search": v}
			} else {
				switch v := v.(type) {
				//			case float64:
				//			case int:
				//			case bool:
				case string:
					queryDoc[k] = bson.M{"$regex": v}
				}
				//queryDoc[k] = v
			}
		}
		data, _ := json.Marshal(queryDoc)
		log.Printf("mongoquery: %s\n", string(data))
	}
	n, err := collection.CountDocuments(ctx, queryDoc, &options.CountOptions{Collation: &options.Collation{Locale: "en", Strength: 2}})
	if err != nil {
		log.Print(err)
		return 0, err
	}
	return int(n), nil
}

//GetEffects getting a list of effects regarding to the query
func (m *MongoDAO) GetEffects(query string, offset int, limit int) (int, []model.Effect, error) {
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(effectsCollectionName)
	queryDoc := bson.M{}
	if query != "" {
		var queryM map[string]interface{}
		err := json.Unmarshal([]byte(query), &queryM)
		if err != nil {
			log.Print(err)
			return 0, nil, err
		}
		for k, v := range queryM {
			if k == "$fulltext" {
				queryDoc["$text"] = bson.M{"$search": v}
			} else {
				switch v := v.(type) {
				//			case float64:
				//			case int:
				//			case bool:
				case string:
					queryDoc[k] = bson.M{"$regex": v}
				}
				//queryDoc[k] = v
			}
		}
		data, _ := json.Marshal(queryDoc)
		log.Printf("mongoquery: %s\n", string(data))
	}
	n, err := collection.CountDocuments(ctx, queryDoc, &options.CountOptions{Collation: &options.Collation{Locale: "en", Strength: 2}})
	if err != nil {
		log.Print(err)
		return 0, nil, err
	}
	cursor, err := collection.Find(ctx, queryDoc, &options.FindOptions{Collation: &options.Collation{Locale: "en", Strength: 2}, Sort: bson.D{{"manufacturer", 1}, {"model", 1}}})
	if err != nil {
		log.Print(err)
		return 0, nil, err
	}
	defer cursor.Close(ctx)
	effects := make([]model.Effect, 0)
	count := 0
	docs := 0
	for cursor.Next(ctx) {
		if count >= offset {
			if docs < limit {
				var effect model.Effect
				if err = cursor.Decode(&effect); err != nil {
					log.Print(err)
					return 0, nil, err
				} else {
					effects = append(effects, effect)
					docs++
				}
			} else {
				break
			}
		}
		count++
	}
	return int(n), effects, nil
}

func (m *MongoDAO) GetEffect(effectID string) (model.Effect, error) {
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(effectsCollectionName)
	objectID, _ := primitive.ObjectIDFromHex(effectID)
	result := collection.FindOne(ctx, bson.M{"_id": objectID})
	err := result.Err()
	if err == mongo.ErrNoDocuments {
		log.Print(err)
		return model.Effect{}, ErrNoDocument
	}
	if err != nil {
		log.Print(err)
		return model.Effect{}, err
	}
	var effect model.Effect
	if err := result.Decode(&effect); err != nil {
		log.Print(err)
		return model.Effect{}, err
	} else {
		return effect, nil
	}
}

// CheckUser checking username and password... returns true if the user is active and the password for this user is correct
func (m *MongoDAO) CheckUser(username string, password string) bool {
	username = strings.ToLower(username)
	pwd, ok := m.users[username]
	if ok {
		if pwd == password {
			return true
		} else {
			user, ok := m.GetUser(username)
			if ok {
				if user.Password == password {
					return true
				}
			}
		}
	}

	if !ok {
		user, ok := m.GetUser(username)
		if ok {
			if user.Password == password {
				return true
			}
		}
	}

	return false
}

// GetUser getting the usermolde
func (m *MongoDAO) GetUser(username string) (model.User, bool) {
	username = strings.ToLower(username)
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	usersCollection := m.database.Collection(usersCollectionName)
	var user model.User
	filter := bson.M{"name": username}
	err := usersCollection.FindOne(ctx, filter).Decode(&user)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
		return model.User{}, false
	}
	password := user.Password
	hash := BuildPasswordHash(password)
	m.users[username] = hash
	return user, true
}

// DropAll dropping all data from the database
func (m *MongoDAO) DropAll() {
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collectionNames, err := m.database.ListCollectionNames(ctx, bson.D{}, &options.ListCollectionsOptions{})
	if err != nil {
		log.Fatal(err)
	}
	for _, name := range collectionNames {
		if name != usersCollectionName {
			collection := m.database.Collection(name)
			err = collection.Drop(ctx)
			if err != nil {
				log.Fatal(err)
			}
		}
	}
}

// Stop stopping the mongodao
func (m *MongoDAO) Stop() {
	m.ticker.Stop()
	m.done <- true
}

// AddUser adding a new user to the system
func (m *MongoDAO) AddUser(user model.User) error {
	if user.Name == "" {
		return errors.New("username should not be empty")
	}
	user.Name = strings.ToLower(user.Name)
	_, ok := m.users[user.Name]
	if ok {
		return errors.New("username already exists")
	}

	user.Password = BuildPasswordHash(user.Password)

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(usersCollectionName)
	_, err := collection.InsertOne(ctx, user)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
		return err
	}
	m.users[user.Name] = user.Password
	return nil
}

// DeleteUser deletes one user from the system
func (m *MongoDAO) DeleteUser(username string) error {
	if username == "" {
		return errors.New("username should not be empty")
	}
	username = strings.ToLower(username)
	_, ok := m.users[username]
	if !ok {
		return errors.New("username not exists")
	}

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(usersCollectionName)
	filter := bson.M{"name": username}
	_, err := collection.DeleteOne(ctx, filter)
	if err != nil {
		fmt.Printf("error: %s\n", err.Error())
		return err
	}
	delete(m.users, username)
	return nil
}

// ChangePWD changes the apssword of a single user
func (m *MongoDAO) ChangePWD(username string, newpassword string, oldpassword string) error {
	if username == "" {
		return errors.New("username should not be empty")
	}
	username = strings.ToLower(username)
	pwd, ok := m.users[username]
	if !ok {
		return errors.New("username not registered")
	}

	newpassword = BuildPasswordHash(newpassword)
	oldpassword = BuildPasswordHash(oldpassword)
	if pwd != oldpassword {
		return errors.New("actual password incorrect")
	}

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(usersCollectionName)
	filter := bson.M{"name": username}
	update := bson.D{{"$set", bson.D{{"password", newpassword}}}}
	result := collection.FindOneAndUpdate(ctx, filter, update)
	if result.Err() != nil {
		fmt.Printf("error: %s\n", result.Err().Error())
		return result.Err()
	}
	m.users[username] = newpassword
	return nil
}

//Backup pinging the mongoDao
func (m *MongoDAO) Backup(path string) error {
	// writung users
	err := m.backupUsers(path)
	if err != nil {
		return err
	}
	// writing tags
	err = m.backupTags(path)
	if err != nil {
		return err
	}
	// writing manufacturers
	err = m.backupManufacturers(path)
	if err != nil {
		return err
	}
	// writing schematics
	err = m.backupSchematics(path)
	if err != nil {
		return err
	}
	// writing effect categories
	err = m.backupEffectTypes(path)
	if err != nil {
		return err
	}
	// writing effects
	err = m.backupEffects(path)
	if err != nil {
		return err
	}
	return err
}

func (m *MongoDAO) backupUsers(path string) error {
	path = path + "/users"
	os.MkdirAll(path, os.ModePerm)

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(usersCollectionName)
	cursor, err := collection.Find(ctx, bson.M{})
	if err != nil {
		log.Fatal(err)
	}
	defer cursor.Close(ctx)
	count := 0
	fmt.Print("backup users: ")
	for cursor.Next(ctx) {
		var user model.User
		if err = cursor.Decode(&user); err != nil {
			log.Fatal(err)
			return err
		} else {
			user.Name = strings.ToLower(user.Name)
			user.Password = BuildPasswordHash(user.Password)
			data, err := json.Marshal(user)
			if err != nil {
				return err
			}
			filename := path + "/" + user.ID.Hex() + ".json"
			ioutil.WriteFile(filename, data, os.ModePerm)
			count++
			if count%100 == 0 {
				fmt.Print(".")
			}
		}
	}
	fmt.Println()
	return nil
}

func (m *MongoDAO) backupTags(path string) error {
	path = path + "/tags"
	os.MkdirAll(path, os.ModePerm)

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(tagsCollectionName)
	cursor, err := collection.Find(ctx, bson.M{})
	if err != nil {
		log.Fatal(err)
	}
	defer cursor.Close(ctx)
	count := 0
	fmt.Print("backup tags: ")
	for cursor.Next(ctx) {
		var model model.Tag
		if err = cursor.Decode(&model); err != nil {
			log.Fatal(err)
			return err
		} else {
			data, err := json.Marshal(model)
			if err != nil {
				return err
			}
			filename := path + "/" + model.ID.Hex() + ".json"
			ioutil.WriteFile(filename, data, os.ModePerm)
			count++
			if count%100 == 0 {
				fmt.Print(".")
			}
		}
	}
	fmt.Println()
	return nil
}

func (m *MongoDAO) backupManufacturers(path string) error {
	path = path + "/manufacturers"
	os.MkdirAll(path, os.ModePerm)

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(manufacturersCollectionName)
	cursor, err := collection.Find(ctx, bson.M{})
	if err != nil {
		log.Fatal(err)
	}
	defer cursor.Close(ctx)
	count := 0
	fmt.Print("backup manufacturers: ")
	for cursor.Next(ctx) {
		var model model.Manufacturer
		if err = cursor.Decode(&model); err != nil {
			log.Fatal(err)
			return err
		} else {
			data, err := json.Marshal(model)
			if err != nil {
				return err
			}
			filename := path + "/" + model.ID.Hex() + ".json"
			ioutil.WriteFile(filename, data, os.ModePerm)
			count++
			if count%100 == 0 {
				fmt.Print(".")
			}
		}
	}
	fmt.Println()
	return nil
}
func (m *MongoDAO) backupSchematics(path string) error {
	path = path + "/schematics"
	os.MkdirAll(path, os.ModePerm)

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(schematicsCollectionName)
	cursor, err := collection.Find(ctx, bson.M{})
	if err != nil {
		log.Fatal(err)
	}
	defer cursor.Close(ctx)
	count := 0
	fmt.Print("backup schematics: ")
	for cursor.Next(ctx) {
		var model model.Schematic
		if err = cursor.Decode(&model); err != nil {
			log.Fatal(err)
			return err
		} else {
			folder := path + "/" + model.ID.Hex()
			os.MkdirAll(folder, os.ModePerm)

			for _, fileid := range model.Files {
				if fileid != "" {

					filename, err := m.GetFilename(fileid)
					if err != nil {
						return err
					}

					imagename := folder + "/" + filename
					file, err := os.Create(imagename)
					if err != nil {
						return err
					}
					defer file.Close()
					err = m.GetFile(fileid, file)
					if err != nil {
						return err
					}
				}
			}
			data, err := json.Marshal(model)
			if err != nil {
				return err
			}

			filename := folder + "/schematic.json"
			ioutil.WriteFile(filename, data, os.ModePerm)
			count++
			if count%100 == 0 {
				fmt.Print(".")
			}
		}
	}
	fmt.Println()
	return nil
}
func (m *MongoDAO) backupEffectTypes(path string) error {
	path = path + "/effecttypes"
	os.MkdirAll(path, os.ModePerm)

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(effectTypesCollectionName)
	cursor, err := collection.Find(ctx, bson.M{})
	if err != nil {
		log.Fatal(err)
	}
	defer cursor.Close(ctx)
	count := 0
	fmt.Print("backup effecttypes: ")
	for cursor.Next(ctx) {
		var model model.EffectType
		if err = cursor.Decode(&model); err != nil {
			log.Fatal(err)
			return err
		} else {
			folder := path + "/" + model.ID.Hex()
			os.MkdirAll(folder, os.ModePerm)

			fileid := model.TypeImage
			if fileid != "" {

				filename, err := m.GetFilename(fileid)
				if err != nil {
					return err
				}

				imagename := folder + "/" + filename
				file, err := os.Create(imagename)
				if err != nil {
					return err
				}
				defer file.Close()
				err = m.GetFile(fileid, file)
				if err != nil {
					return err
				}
				model.TypeImage = filename
			}
			data, err := json.Marshal(model)
			if err != nil {
				return err
			}

			filename := folder + "/effecttype.json"
			ioutil.WriteFile(filename, data, os.ModePerm)
			count++
			if count%100 == 0 {
				fmt.Print(".")
			}
		}
	}
	fmt.Println()
	return nil
}
func (m *MongoDAO) backupEffects(path string) error {
	path = path + "/effects"
	os.MkdirAll(path, os.ModePerm)

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	collection := m.database.Collection(effectsCollectionName)
	cursor, err := collection.Find(ctx, bson.M{})
	if err != nil {
		log.Fatal(err)
	}
	defer cursor.Close(ctx)
	count := 0
	fmt.Print("backup effects: ")
	for cursor.Next(ctx) {
		var model model.Effect
		if err = cursor.Decode(&model); err != nil {
			log.Fatal(err)
			return err
		} else {
			folder := path + "/" + model.ID.Hex()
			os.MkdirAll(folder, os.ModePerm)
			fileid := model.Image
			if fileid != "" {
				filename, err := m.GetFilename(fileid)
				if err != nil {
					return err
				}

				imagename := folder + "/" + filename
				file, err := os.Create(imagename)
				if err != nil {
					return err
				}
				defer file.Close()
				err = m.GetFile(fileid, file)
				if err != nil {
					return err
				}
				model.Image = filename
			}
			data, err := json.Marshal(model)
			if err != nil {
				return err
			}

			filename := folder + "/effect.json"
			ioutil.WriteFile(filename, data, os.ModePerm)
			count++
			if count%100 == 0 {
				fmt.Print(".")
			}
		}
	}
	fmt.Println()
	return nil
}

// Ping pinging the mongoDao
func (m *MongoDAO) Ping() error {
	if !m.initialised {
		return errors.New("mongo client not initialised")
	}
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	return m.database.Client().Ping(ctx, nil)
}