mirror of
https://github.com/gravitl/netmaker.git
synced 2026-04-23 00:17:10 +08:00
292af315dd
* feat(go): add user schema; * feat(go): migrate to user schema; * feat(go): add audit fields; * feat(go): remove unused fields from the network model; * feat(go): add network schema; * feat(go): migrate to network schema; * refactor(go): add comment to clarify migration logic; * fix(go): test failures; * fix(go): test failures; * feat(go): change membership table to store memberships at all scopes; * feat(go): add schema for access grants; * feat(go): remove nameservers from new networks table; ensure db passed for schema functions; * feat(go): set max conns for sqlite to 1; * fix(go): issues updating user account status; * NM-236: streamline operations in HA mode * NM-236: only master pod should subscribe to updates from clients * refactor(go): remove converters and access grants; * refactor(go): add json tags in schema models; * refactor(go): rename file to migrate_v1_6_0.go; * refactor(go): add user groups and user roles tables; use schema tables; * refactor(go): inline get and list from schema package; * refactor(go): inline get network and list users from schema package; * fix(go): staticcheck issues; * fix(go): remove test not in use; fix test case; * fix(go): validate network; * fix(go): resolve static checks; * fix(go): new models errors; * fix(go): test errors; * fix(go): handle no records; * fix(go): add validations for user object; * fix(go): set correct extclient status; * fix(go): test error; * feat(go): make schema the base package; * feat(go): add host schema; * feat(go): use schema host everywhere; * feat(go): inline get host, list hosts and delete host; * feat(go): use non-ptr value; * feat(go): use save to upsert all fields; * feat(go): use save to upsert all fields; * feat(go): save turn endpoint as string; * feat(go): check for gorm error record not found; * fix(go): test failures; * fix(go): update all network fields; * fix(go): update all network fields; * feat(go): add paginated list networks api; * feat(go): add paginated list users api; * feat(go): add paginated list hosts api; * feat(go): add pagination to list groups api; * fix(go): comment; * fix(go): implement marshal and unmarshal text for custom types; * fix(go): implement marshal and unmarshal json for custom types; * fix(go): just use the old model for unmarshalling; * fix(go): implement marshal and unmarshal json for custom types; * NM-271:Import swap: compress/gzip replaced with github.com/klauspost/compress/gzip (2-4x faster, wire-compatible output). Added sync import. Two sync.Pool variables (gzipWriterPool, bufferPool): reuse gzip.Writer and bytes.Buffer across calls instead of allocating fresh ones per publish. compressPayload rewritten: pulls writer + buffer from pools, resets them, compresses at gzip.BestSpeed (level 1), copies the result out of the pooled buffer, and returns both objects to the pools. * feat(go): remove paginated list networks api; * feat(go): use custom paginated response object; * NM-271: Improve server scalability under high host count - Replace stdlib compress/gzip with klauspost/compress at BestSpeed and pool gzip writers and buffers via sync.Pool to eliminate compression as the dominant CPU hotspot. - Debounce peer update broadcasts with a 500ms resettable window capped at 3s max-wait, coalescing rapid-fire PublishPeerUpdate calls into a single broadcast cycle. - Cache HostPeerInfo (batch-refreshed by debounce worker) and HostPeerUpdate (stored as side-effect of each publish) so the pull API and peer_info API serve from pre-computed maps instead of triggering expensive per-host computations under thundering herd conditions. - Warm both caches synchronously at startup before the first publish cycle so early pull requests are served instantly. - Bound concurrent MQTT publishes to 5 via semaphore to prevent broker TCP buffer overflows that caused broken pipe disconnects. - Remove manual Disconnect+SetupMQTT from ConnectionLostHandler and rely on the paho client's built-in AutoReconnect; add a 5s retry wait in publish() to ride out brief reconnection windows. * NM-271: Reduce server CPU contention under high concurrent load - Cache ServerSettings with atomic.Value to eliminate repeated DB reads on every pull request (was 32+ goroutines blocked on read lock) - Batch UpdateNodeCheckin writes in memory, flush every 30s to reduce per-checkin write lock contention (was 88+ goroutines blocked) - Enable SQLite WAL mode + busy_timeout and remove global dbMutex; let SQLite handle concurrency natively (reads no longer block writes) - Move ResetFailedOverPeer/ResetAutoRelayedPeer to async in pull() handler since results don't affect the cached response - Skip no-op UpsertNode writes in failover/relay reset functions (early return when node has no failover/relay state) - Remove CheckHostPorts from hostUpdateFallback hot path - Switch to pure-Go SQLite driver (glebarez/sqlite), set CGO_ENABLED=0 * fix(go): ensure default values for page and per_page are used when not passed; * fix(go): rename v1.6.0 to v1.5.1; * fix(go): check for gorm.ErrRecordNotFound instead of database.IsEmptyRecord; * fix(go): use host id, not pending host id; * NM-271: Revert pure-Go SQLite and FIPS disable to verify impact Revert to CGO-based mattn/go-sqlite3 driver and re-enable FIPS to isolate whether these changes are still needed now that the global dbMutex has been removed and WAL mode is enabled. Keep WAL mode pragma with mattn-compatible DSN format. * feat(go): add filters to paginated apis; * feat(go): add filters to paginated apis; * feat(go): remove check for max username length; * feat(go): add filters to count as well; * feat(go): use library to check email address validity; * feat(go): ignore pagination if params not passed; * fix(go): pagination issues; * fix(go): check exists before using; * fix(go): remove debug log; * NM-271: rm debug logs * NM-271: check if caching is enabled * NM-271: add server sync mq topic for HA mode * NM-271: fix build * NM-271: push metrics in batch to exproter over api * NM-271: use basic auth for exporter metrics api * fix(go): use gorm err record not found; * NM-271: Add monitoring stack on demand * NM-271: -m arg for install script should only add monitoring stack * fix(go): use gorm err record not found; * NM-271: update docker compose file for prometheus * NM-271: update docker compose file for prometheus * fix(go): use user principal name when creating pending user; * fix(go): use schema package for consts; * NM-236: rm duplicate network hook * NM-271: add server topic to reset idp hooks on master node * fix(go): prevent disabling superadmin user; Co-authored-by: tenki-reviewer[bot] <262613592+tenki-reviewer[bot]@users.noreply.github.com> * fix(go): swap is admin and is superadmin; Co-authored-by: tenki-reviewer[bot] <262613592+tenki-reviewer[bot]@users.noreply.github.com> * fix(go): remove dead code block; https://github.com/gravitl/netmaker/pull/3910#discussion_r2928837937 * fix(go): incorrect message when trying to disable self; https://github.com/gravitl/netmaker/pull/3910#discussion_r2928837934 * NM-271: fix stale peers on reset_failovered pull and add HTTP timeout to metrics exporter Run the failover/relay reset synchronously in the pull handler so the response reflects post-reset topology instead of serving stale cached peers. Add a 30s timeout to the metrics exporter HTTP client to prevent PushAllMetricsToExporter from blocking the Keepalive loop. * NM-271: fix gzip pool corruption, MQTT topic mismatch, stale settings cache, and reduce redundant DB fetches - Only return gzip.Writer to pool after successful Close to prevent silently malformed MQTT payloads from a previously errored writer. - Fix serversync subscription to exact topic match since syncType is now in the message payload, not the topic path. - Prevent zero-value ServerSettings from being cached indefinitely when the DB record is missing or unmarshal fails on startup. - Return fetched hosts/nodes from RefreshHostPeerInfoCache so warmPeerCaches reuses them instead of querying the DB twice. - Compute fresh HostPeerUpdate on reset_failovered pull instead of serving stale cache, and store result back for subsequent requests. * NM-271: fix gzip writer pool leak, log checkin flush errors, and fix master pod ordinal parsing - Reset gzip.Writer to io.Discard before returning to pool so errored writers are never leaked or silently reused with corrupt state. - Track and log failed DB inserts in FlushNodeCheckins so operators have visibility when check-in timestamps are lost. - Parse StatefulSet pod ordinal as integer instead of using HasSuffix to prevent netmaker-10 from being misidentified as master pod. * NM-271: simplify masterpod logic * fix(go): use correct header; Co-authored-by: tenki-reviewer[bot] <262613592+tenki-reviewer[bot]@users.noreply.github.com> * fix(go): return after error response; Co-authored-by: tenki-reviewer[bot] <262613592+tenki-reviewer[bot]@users.noreply.github.com> * fix(go): use correct order of params; https://github.com/gravitl/netmaker/pull/3910#discussion_r2929593036 * fix(go): set default values for page and page size; use v2 instead of /list; * NM-271: use host name * Update mq/serversync.go Co-authored-by: tenki-reviewer[bot] <262613592+tenki-reviewer[bot]@users.noreply.github.com> * NM-271: fix duplicate serversynce case * NM-271: streamline gw updates * Update logic/auth.go Co-authored-by: tenki-reviewer[bot] <262613592+tenki-reviewer[bot]@users.noreply.github.com> * Update schema/user_roles.go Co-authored-by: tenki-reviewer[bot] <262613592+tenki-reviewer[bot]@users.noreply.github.com> * fix(go): syntax error; * fix(go): set default values when page and per_page are not passed or 0; * fix(go): use uuid.parse instead of uuid.must parse; * fix(go): review errors; * fix(go): review errors; * Update controllers/user.go Co-authored-by: tenki-reviewer[bot] <262613592+tenki-reviewer[bot]@users.noreply.github.com> * Update controllers/user.go Co-authored-by: tenki-reviewer[bot] <262613592+tenki-reviewer[bot]@users.noreply.github.com> * NM-163: fix errors: * Update db/types/options.go Co-authored-by: tenki-reviewer[bot] <262613592+tenki-reviewer[bot]@users.noreply.github.com> * fix(go): persist return user in event; * Update db/types/options.go Co-authored-by: tenki-reviewer[bot] <262613592+tenki-reviewer[bot]@users.noreply.github.com> * NM-271: signal pull on ip changes * NM-163: duplicate lines of code * NM-163: fix(go): fix missing return and filter parsing in user controller - Add missing return after error response in updateUserAccountStatus to prevent double-response and spurious ext-client side-effects - Use switch statements in listUsers to skip unrecognized account_status and mfa_status filter values * NM-271: signal pull req on node ip change * fix(go): check for both min and max page size; * NM-271: refresh node object before update * fix(go): enclose transfer superadmin in transaction; * fix(go): review errors; * fix(go): remove free tier checks; * fix(go): review fixes; * NM-271: streamline ip pool ops * NM-271: fix tests, set max idle conns * NM-271: fix(go): fix data races in settings cache and peer update worker - Use pointer type in atomic.Value for serverSettingsCache to avoid replacing the variable non-atomically in InvalidateServerSettingsCache - Swap peerUpdateReplace flag before draining the channel to prevent a concurrent replacePeers=true from being consumed by the wrong cycle --------- Co-authored-by: VishalDalwadi <dalwadivishal26@gmail.com> Co-authored-by: Vishal Dalwadi <51291657+VishalDalwadi@users.noreply.github.com> Co-authored-by: tenki-reviewer[bot] <262613592+tenki-reviewer[bot]@users.noreply.github.com>
148 lines
6.4 KiB
Go
148 lines
6.4 KiB
Go
// Environment file for getting variables
|
|
// Currently the only thing it does is set the master password
|
|
// Should probably have it take over functions from OS such as port and mongodb connection details
|
|
// Reads from the config/environments/dev.yaml file by default
|
|
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"time"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
// setting dev by default
|
|
func getEnv() string {
|
|
env := os.Getenv("NETMAKER_ENV")
|
|
if len(env) == 0 {
|
|
return "dev"
|
|
}
|
|
return env
|
|
}
|
|
|
|
// Config : application config stored as global variable
|
|
var Config *EnvironmentConfig = &EnvironmentConfig{}
|
|
var SetupErr error
|
|
|
|
// EnvironmentConfig - environment conf struct
|
|
type EnvironmentConfig struct {
|
|
Server ServerConfig `yaml:"server"`
|
|
SQL SQLConfig `yaml:"sql"`
|
|
ClickHouse ClickHouseConfig `yaml:"clickhouse"`
|
|
}
|
|
|
|
// ServerConfig - server conf struct
|
|
type ServerConfig struct {
|
|
CoreDNSAddr string `yaml:"corednsaddr"`
|
|
APIConnString string `yaml:"apiconn"`
|
|
APIHost string `yaml:"apihost"`
|
|
APIPort string `yaml:"apiport"`
|
|
Broker string `yaml:"broker"`
|
|
ServerBrokerEndpoint string `yaml:"serverbrokerendpoint"`
|
|
BrokerType string `yaml:"brokertype"`
|
|
EmqxRestEndpoint string `yaml:"emqxrestendpoint"`
|
|
NetclientAutoUpdate string `yaml:"netclientautoupdate"`
|
|
NetclientEndpointDetection string `yaml:"netclientendpointdetection"`
|
|
MasterKey string `yaml:"masterkey"`
|
|
DNSKey string `yaml:"dnskey"`
|
|
AllowedOrigin string `yaml:"allowedorigin"`
|
|
HostName string `yaml:"host_name"`
|
|
RestBackend string `yaml:"restbackend"`
|
|
MessageQueueBackend string `yaml:"messagequeuebackend"`
|
|
DNSMode string `yaml:"dnsmode"`
|
|
DisableRemoteIPCheck string `yaml:"disableremoteipcheck"`
|
|
Version string `yaml:"version"`
|
|
SQLConn string `yaml:"sqlconn"`
|
|
Platform string `yaml:"platform"`
|
|
Database string `yaml:"database"`
|
|
Verbosity int32 `yaml:"verbosity"`
|
|
AuthProvider string `yaml:"authprovider"`
|
|
OIDCIssuer string `yaml:"oidcissuer"`
|
|
ClientID string `yaml:"clientid"`
|
|
ClientSecret string `yaml:"clientsecret"`
|
|
FrontendURL string `yaml:"frontendurl"`
|
|
DisplayKeys string `yaml:"displaykeys"`
|
|
AzureTenant string `yaml:"azuretenant"`
|
|
Telemetry string `yaml:"telemetry"`
|
|
HostNetwork string `yaml:"hostnetwork"`
|
|
Server string `yaml:"server"`
|
|
PublicIPService string `yaml:"publicipservice"`
|
|
MQPassword string `yaml:"mqpassword"`
|
|
MQUserName string `yaml:"mqusername"`
|
|
MetricsExporter string `yaml:"metrics_exporter"`
|
|
BasicAuth string `yaml:"basic_auth"`
|
|
LicenseValue string `yaml:"license_value"`
|
|
NetmakerTenantID string `yaml:"netmaker_tenant_id"`
|
|
IsPro string `yaml:"is_ee" json:"IsEE"`
|
|
StunPort int `yaml:"stun_port"`
|
|
TurnServer string `yaml:"turn_server"`
|
|
TurnApiServer string `yaml:"turn_api_server"`
|
|
TurnPort int `yaml:"turn_port"`
|
|
TurnUserName string `yaml:"turn_username"`
|
|
TurnPassword string `yaml:"turn_password"`
|
|
UseTurn bool `yaml:"use_turn"`
|
|
UsersLimit int `yaml:"user_limit"`
|
|
NetworksLimit int `yaml:"network_limit"`
|
|
MachinesLimit int `yaml:"machines_limit"`
|
|
IngressesLimit int `yaml:"ingresses_limit"`
|
|
EgressesLimit int `yaml:"egresses_limit"`
|
|
DeployedByOperator bool `yaml:"deployed_by_operator"`
|
|
Environment string `yaml:"environment"`
|
|
JwtValidityDuration time.Duration `yaml:"jwt_validity_duration" swaggertype:"primitive,integer" format:"int64"`
|
|
JwtValidityDurationClients time.Duration `yaml:"jwt_validity_duration_clients" swaggertype:"primitive,integer" format:"int64"`
|
|
RacRestrictToSingleNetwork bool `yaml:"rac_restrict_to_single_network"`
|
|
CacheEnabled string `yaml:"caching_enabled"`
|
|
EndpointDetection bool `yaml:"endpoint_detection"`
|
|
AllowedEmailDomains string `yaml:"allowed_email_domains"`
|
|
EmailSenderAddr string `yaml:"email_sender_addr"`
|
|
EmailSenderUser string `yaml:"email_sender_user"`
|
|
EmailSenderPassword string `yaml:"email_sender_password"`
|
|
SmtpHost string `yaml:"smtp_host"`
|
|
SmtpPort int `yaml:"smtp_port"`
|
|
MetricInterval string `yaml:"metric_interval"`
|
|
MetricsPort int `yaml:"metrics_port"`
|
|
ManageDNS bool `yaml:"manage_dns"`
|
|
Stun bool `yaml:"stun"`
|
|
StunServers string `yaml:"stun_servers"`
|
|
DefaultDomain string `yaml:"default_domain"`
|
|
PublicIp string `yaml:"public_ip"`
|
|
}
|
|
|
|
// SQLConfig - Generic SQL Config
|
|
type SQLConfig struct {
|
|
Host string `yaml:"host"`
|
|
Port int32 `yaml:"port"`
|
|
Username string `yaml:"username"`
|
|
Password string `yaml:"password"`
|
|
DB string `yaml:"db"`
|
|
SSLMode string `yaml:"sslmode"`
|
|
}
|
|
|
|
type ClickHouseConfig struct {
|
|
Host string `yaml:"host"`
|
|
Port int32 `yaml:"port"`
|
|
Database string `yaml:"database"`
|
|
Username string `yaml:"username"`
|
|
Password string `yaml:"password"`
|
|
}
|
|
|
|
// reading in the env file
|
|
func ReadConfig(absolutePath string) (*EnvironmentConfig, error) {
|
|
if len(absolutePath) == 0 {
|
|
absolutePath = fmt.Sprintf("environments/%s.yaml", getEnv())
|
|
}
|
|
f, err := os.Open(absolutePath)
|
|
var cfg EnvironmentConfig
|
|
if err != nil {
|
|
return &cfg, err
|
|
}
|
|
defer f.Close()
|
|
|
|
decoder := yaml.NewDecoder(f)
|
|
if decoder.Decode(&cfg) != nil {
|
|
return &cfg, err
|
|
}
|
|
return &cfg, err
|
|
}
|