Merge branch 'release-v1.5.1' of https://github.com/gravitl/netmaker into NM-311

controllers/hosts.go

@@ -698,25 +698,25 @@ func bulkDeleteHosts(w http.ResponseWriter, r *http.Request) {
 		for _, idStr := range req.IDs {
 			hostID, err := uuid.Parse(idStr)
 			if err != nil {
-				slog.Error("bulk host delete: invalid host id", "id", idStr)
+				slog.Debug("bulk host delete: invalid host id", "id", idStr)
 				continue
 			}
 			currHost := &schema.Host{ID: hostID}
 			if err = currHost.Get(db.WithContext(context.Background())); err != nil {
-				slog.Error("bulk host delete: host not found", "id", idStr, "error", err)
+				slog.Debug("bulk host delete: host not found", "id", idStr, "error", err)
 				continue
 			}
 			var hostNodes []models.Node
 			for _, nodeID := range currHost.Nodes {
 				node, err := logic.GetNodeByID(nodeID)
 				if err != nil {
-					slog.Error("bulk host delete: failed to get node", "nodeid", nodeID, "error", err)
+					slog.Debug("bulk host delete: failed to get node", "nodeid", nodeID, "error", err)
 					continue
 				}
 				hostNodes = append(hostNodes, node)
 			}
 			if err = logic.RemoveHost(currHost, true); err != nil {
-				slog.Error("bulk host delete: failed to remove host", "id", idStr, "error", err)
+				slog.Debug("bulk host delete: failed to remove host", "id", idStr, "error", err)
 				continue
 			}
 			for _, node := range hostNodes {
@@ -724,14 +724,14 @@ func bulkDeleteHosts(w http.ResponseWriter, r *http.Request) {
 			}
 			if servercfg.GetBrokerType() == servercfg.EmqxBrokerType {
 				if err := mq.GetEmqxHandler().DeleteEmqxUser(currHost.ID.String()); err != nil {
-					slog.Error("bulk host delete: failed to remove EMQX credentials", "id", currHost.ID, "error", err)
+					slog.Debug("bulk host delete: failed to remove EMQX credentials", "id", currHost.ID, "error", err)
 				}
 			}
 			if err = mq.HostUpdate(&models.HostUpdate{
 				Action: models.DeleteHost,
 				Host:   *currHost,
 			}); err != nil {
-				slog.Error("bulk host delete: failed to send host update", "id", currHost.ID, "error", err)
+				slog.Debug("bulk host delete: failed to send host update", "id", currHost.ID, "error", err)
 			}
 			(&schema.PendingHost{HostID: currHost.ID.String()}).DeleteAllPendingHosts(db.WithContext(context.TODO()))
 			logic.LogEvent(&models.Event{

controllers/server.go

@@ -337,7 +337,8 @@ func reInit(curr, new models.ServerSettings, force bool) {
 	// On force AutoUpdate change, change AutoUpdate for all hosts.
 	// On force FlowLogs enable, enable FlowLogs for all hosts.
 	// On FlowLogs disable, forced or not, disable FlowLogs for all hosts.
-	if force || !new.EnableFlowLogs {
+	// On NetclientAutoUpdate disable, forced or not, disable AutoUpdate for all hosts.
+	if force || !new.EnableFlowLogs || !new.NetclientAutoUpdate {
 		if curr.NetclientAutoUpdate != new.NetclientAutoUpdate ||
 			curr.EnableFlowLogs != new.EnableFlowLogs {
 			hosts, _ := (&schema.Host{}).ListAll(db.WithContext(context.TODO()))

db/sqlite.go

@@ -62,6 +62,7 @@ func (s *sqliteConnector) connect() (*gorm.DB, error) {
 		return nil, err
 	}
 
+	//sqlDB.SetMaxOpenConns(1)
 	sqlDB.SetMaxIdleConns(1)
 
 	return db, nil

logic/extpeers.go

@@ -684,6 +684,7 @@ func getExtPeerEgressRoute(node models.Node, extPeer models.ExtClient) (egressRo
 		NodeAddr:      node.Address,
 		NodeAddr6:     node.Address6,
 		EgressRanges:  extPeer.ExtraAllowedIPs,
+		Network:       node.Network,
 	}
 	for _, extraAllowedIP := range extPeer.ExtraAllowedIPs {
 		r.EgressRangesWithMetric = append(r.EgressRangesWithMetric, models.EgressRangeMetric{

logic/peers.go

@@ -238,14 +238,14 @@ func computeHostPeerInfo(host *schema.Host, allNodes []models.Node, serverInfo m
 
 // GetPeerUpdateForHost - gets the consolidated peer update for the host from all networks
 func GetPeerUpdateForHost(network string, host *schema.Host, allNodes []models.Node,
-	deletedNode *models.Node, deletedClients []models.ExtClient) (models.HostPeerUpdate, error) {
+	deletedNode *models.Node, deletedClients []models.ExtClient) (hostPeerUpdate models.HostPeerUpdate, err error) {
 	if host == nil {
 		return models.HostPeerUpdate{}, errors.New("host is nil")
 	}
 
 	// track which nodes are deleted
 	// after peer calculation, if peer not in list, add delete config of peer
-	hostPeerUpdate := models.HostPeerUpdate{
+	hostPeerUpdate = models.HostPeerUpdate{
 		Host:          *host,
 		Server:        servercfg.GetServer(),
 		ServerVersion: servercfg.GetVersion(),
@@ -266,6 +266,9 @@ func GetPeerUpdateForHost(network string, host *schema.Host, allNodes []models.N
 		GwNodes:            make(map[schema.NetworkID][]models.Node),
 		AddressIdentityMap: make(map[string]models.PeerIdentity),
 	}
+	defer func() {
+		hostPeerUpdate.EgressRoutes = deduplicateEgressRoutes(hostPeerUpdate.EgressRoutes)
+	}()
 	if host.DNS == "no" {
 		hostPeerUpdate.ManageDNS = false
 	}
@@ -931,6 +934,18 @@ func getNodeAllowedIPs(peer, node *models.Node) []net.IPNet {
 	}
 	return allowedips
 }
+func deduplicateEgressRoutes(routes []models.EgressNetworkRoutes) []models.EgressNetworkRoutes {
+	seen := make(map[string]struct{}, len(routes))
+	result := make([]models.EgressNetworkRoutes, 0, len(routes))
+	for _, r := range routes {
+		key := r.PeerKey + "|" + r.Network
+		if _, exists := seen[key]; !exists {
+			seen[key] = struct{}{}
+			result = append(result, r)
+		}
+	}
+	return result
+}
 
 func getCIDRMaskFromAddr(addr string) net.IPMask {
 	cidr := net.CIDRMask(32, 32)

migrate/migrate.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"net"
 	"slices"
+	"strings"
 	"time"
 
 	"golang.org/x/exp/slog"
@@ -589,6 +590,9 @@ func migrateToEgressV1() {
 					CreatedBy: user.UserName,
 					CreatedAt: time.Now().UTC(),
 				}
+				if !e.Nat {
+					e.Mode = schema.DisabledNAT
+				}
 				err = e.Create(db.WithContext(context.TODO()))
 				if err == nil {
 					acl := models.Acl{
@@ -838,42 +842,53 @@ func migrateNameservers() {
 		if !node.IsGw {
 			continue
 		}
+
 		if node.IngressDNS != "" {
-			if (node.Address.IP != nil && node.Address.IP.String() == node.IngressDNS) ||
+			var nsIPs []string
-				(node.Address6.IP != nil && node.Address6.IP.String() == node.IngressDNS) {
+			for _, nsIP := range strings.Split(node.IngressDNS, ",") {
-				continue
+				nsIP = strings.TrimSpace(nsIP)
+
+				if (node.Address.IP != nil && node.Address.IP.String() == nsIP) ||
+					(node.Address6.IP != nil && node.Address6.IP.String() == nsIP) {
+					continue
+				}
+				if nsIP == "8.8.8.8" || nsIP == "1.1.1.1" || nsIP == "9.9.9.9" {
+					continue
+				}
+
+				nsIPs = append(nsIPs, nsIP)
 			}
-			if node.IngressDNS == "8.8.8.8" || node.IngressDNS == "1.1.1.1" || node.IngressDNS == "9.9.9.9" {
+
-				continue
+			if len(nsIPs) > 0 {
-			}
+				host := &schema.Host{
-			host := &schema.Host{
+					ID: node.HostID,
-				ID: node.HostID,
+				}
-			}
+				err := host.Get(db.WithContext(context.TODO()))
-			err := host.Get(db.WithContext(context.TODO()))
+				if err != nil {
-			if err != nil {
+					continue
-				continue
+				}
-			}
+				ns := schema.Nameserver{
-			ns := schema.Nameserver{
+					ID:        uuid.NewString(),
-				ID:        uuid.NewString(),
+					Name:      fmt.Sprintf("%s gw nameservers", host.Name),
-				Name:      fmt.Sprintf("%s gw nameservers", host.Name),
+					NetworkID: node.Network,
-				NetworkID: node.Network,
+					Servers:   nsIPs,
-				Servers:   []string{node.IngressDNS},
+					MatchAll:  true,
-				MatchAll:  true,
+					Domains: []schema.NameserverDomain{
-				Domains: []schema.NameserverDomain{
+						{
-					{
+							Domain: ".",
-						Domain: ".",
+						},
 					},
-				},
+					Nodes: datatypes.JSONMap{
-				Nodes: datatypes.JSONMap{
+						node.ID.String(): struct{}{},
-					node.ID.String(): struct{}{},
+					},
-				},
+					Tags:      make(datatypes.JSONMap),
-				Tags:      make(datatypes.JSONMap),
+					Status:    true,
-				Status:    true,
+					CreatedBy: superAdmin.Username,
-				CreatedBy: superAdmin.Username,
+				}
+				_ = ns.Create(db.WithContext(context.TODO()))
+				node.IngressDNS = ""
+				_ = logic.UpsertNode(&node)
 			}
-			_ = ns.Create(db.WithContext(context.TODO()))
-			node.IngressDNS = ""
-			_ = logic.UpsertNode(&node)
 		}
 	}
 }

migrate/migrate_v1_5_1.go

@@ -255,12 +255,19 @@ func migrateNetworks(ctx context.Context) error {
 			}
 
 			for _, nsIP := range network.NameServers {
-				if net.ParseIP(nsIP) == nil {
+				ip := net.ParseIP(nsIP)
+				if ip == nil {
 					continue
 				}
-				if (cidr != nil && !cidr.Contains(net.ParseIP(nsIP))) &&
+
-					(cidrv6 != nil && !cidrv6.Contains(net.ParseIP(nsIP))) {
+				if ip.To4() != nil {
-					ns.Servers = append(ns.Servers, nsIP)
+					if cidr != nil && !cidr.Contains(ip) {
+						ns.Servers = append(ns.Servers, nsIP)
+					}
+				} else {
+					if cidrv6 != nil && !cidrv6.Contains(ip) {
+						ns.Servers = append(ns.Servers, nsIP)
+					}
 				}
 			}
 
@@ -405,10 +412,6 @@ func migrateHosts(ctx context.Context) error {
 			}
 		}
 
-		if _host.IsDefault && !_host.AutoUpdate {
-			_host.AutoUpdate = true
-		}
-
 		logger.Log(4, fmt.Sprintf("migrating host %s", _host.ID))
 
 		err = _host.Create(ctx)

pro/license.go

@@ -140,7 +140,8 @@ func ValidateLicense() (err error) {
 	proLogic.SetFeatureFlags(licenseResponse.FeatureFlags)
 	proLogic.SetDeploymentMode(licenseResponse.DeploymentMode)
 
-	_ = mq.PublishExporterFeatureFlags()
+	go mq.PublishExporterFeatureFlags()
+	go mq.PublishPeerUpdate(false)
 
 	slog.Info("License validation succeeded!")
 	return nil

schema/egress.go

@@ -13,8 +13,9 @@ const egressTable = "egresses"
 type EgressNATMode string
 
 const (
-	VirtualNAT EgressNATMode = "virtual_nat"
+	DisabledNAT EgressNATMode = "disabled"
-	DirectNAT  EgressNATMode = "direct_nat"
+	VirtualNAT  EgressNATMode = "virtual_nat"
+	DirectNAT   EgressNATMode = "direct_nat"
 )
 
 type Egress struct {