Merge pull request #3967 from gravitl/fix/max-conns

Set max open connections to 1, add write mutex to sqlite ops

database/sqlite.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"errors"
+	"sync"
 	"time"
 
 	"github.com/gravitl/netmaker/db"
@@ -13,6 +14,9 @@ import (
 // SqliteDB is the db object for sqlite database connections
 var SqliteDB *sql.DB
 
+// sqliteWriteMu serializes SQLite write operations to reduce lock contention.
+var sqliteWriteMu sync.Mutex
+
 // SQLITE_FUNCTIONS - contains a map of the functions for sqlite
 var SQLITE_FUNCTIONS = map[string]interface{}{
 	INIT_DB:      initSqliteDB,
@@ -40,6 +44,9 @@ func initSqliteDB() error {
 }
 
 func sqliteCreateTable(tableName string) error {
+	sqliteWriteMu.Lock()
+	defer sqliteWriteMu.Unlock()
+
 	statement, err := SqliteDB.Prepare("CREATE TABLE IF NOT EXISTS " + tableName + " (key TEXT NOT NULL UNIQUE PRIMARY KEY, value TEXT)")
 	if err != nil {
 		return err
@@ -54,6 +61,9 @@ func sqliteCreateTable(tableName string) error {
 
 func sqliteInsert(key string, value string, tableName string) error {
 	if key != "" && value != "" {
+		sqliteWriteMu.Lock()
+		defer sqliteWriteMu.Unlock()
+
 		insertSQL := "INSERT OR REPLACE INTO " + tableName + " (key, value) VALUES (?, ?)"
 		statement, err := SqliteDB.Prepare(insertSQL)
 		if err != nil {
@@ -81,6 +91,9 @@ func sqliteInsertPeer(key string, value string) error {
 }
 
 func sqliteDeleteRecord(tableName string, key string) error {
+	sqliteWriteMu.Lock()
+	defer sqliteWriteMu.Unlock()
+
 	deleteSQL := "DELETE FROM " + tableName + " WHERE key = ?"
 	statement, err := SqliteDB.Prepare(deleteSQL)
 	if err != nil {
@@ -94,6 +107,9 @@ func sqliteDeleteRecord(tableName string, key string) error {
 }
 
 func sqliteDeleteAllRecords(tableName string) error {
+	sqliteWriteMu.Lock()
+	defer sqliteWriteMu.Unlock()
+
 	deleteSQL := "DELETE FROM " + tableName
 	statement, err := SqliteDB.Prepare(deleteSQL)
 	if err != nil {

db/sqlite.go

@@ -62,7 +62,7 @@ func (s *sqliteConnector) connect() (*gorm.DB, error) {
 		return nil, err
 	}
 
-	//sqlDB.SetMaxOpenConns(1)
+	sqlDB.SetMaxOpenConns(1)
 	sqlDB.SetMaxIdleConns(1)
 
 	return db, nil

migrate/migrate_v1_5_1.go

@@ -93,7 +93,7 @@ func migrateV1_5_1(ctx context.Context) error {
 }
 
 func migrateUsers(ctx context.Context) error {
-	records, err := database.FetchRecords(database.USERS_TABLE_NAME)
+	records, err := FetchAll(ctx, database.USERS_TABLE_NAME)
 	if err != nil && !database.IsEmptyRecord(err) {
 		return err
 	}
@@ -147,7 +147,7 @@ func migrateUsers(ctx context.Context) error {
 }
 
 func migrateNetworks(ctx context.Context) error {
-	records, err := database.FetchRecords(database.NETWORKS_TABLE_NAME)
+	records, err := FetchAll(ctx, database.NETWORKS_TABLE_NAME)
 	if err != nil && !database.IsEmptyRecord(err) {
 		return err
 	}
@@ -286,7 +286,7 @@ func migrateNetworks(ctx context.Context) error {
 }
 
 func migrateUserRoles(ctx context.Context) error {
-	records, err := database.FetchRecords(database.USER_PERMISSIONS_TABLE_NAME)
+	records, err := FetchAll(ctx, database.USER_PERMISSIONS_TABLE_NAME)
 	if err != nil && !database.IsEmptyRecord(err) {
 		return err
 	}
@@ -311,7 +311,7 @@ func migrateUserRoles(ctx context.Context) error {
 }
 
 func migrateUserGroups(ctx context.Context) error {
-	records, err := database.FetchRecords(database.USER_GROUPS_TABLE_NAME)
+	records, err := FetchAll(ctx, database.USER_GROUPS_TABLE_NAME)
 	if err != nil && !database.IsEmptyRecord(err) {
 		return err
 	}
@@ -336,7 +336,7 @@ func migrateUserGroups(ctx context.Context) error {
 }
 
 func migrateHosts(ctx context.Context) error {
-	records, err := database.FetchRecords(database.HOSTS_TABLE_NAME)
+	records, err := FetchAll(ctx, database.HOSTS_TABLE_NAME)
 	if err != nil && !database.IsEmptyRecord(err) {
 		return err
 	}
@@ -423,3 +423,22 @@ func migrateHosts(ctx context.Context) error {
 
 	return nil
 }
+
+func FetchAll(ctx context.Context, tableName string) (map[string]string, error) {
+	row, err := db.FromContext(ctx).Raw("SELECT * FROM " + tableName + " ORDER BY key").Rows()
+	if err != nil {
+		return nil, err
+	}
+	records := make(map[string]string)
+	defer row.Close()
+	for row.Next() { // Iterate and fetch the records from result cursor
+		var key string
+		var value string
+		row.Scan(&key, &value)
+		records[key] = value
+	}
+	if len(records) == 0 {
+		return nil, gorm.ErrRecordNotFound
+	}
+	return records, nil
+}