diff --git a/backend/internal/api/config.go b/backend/internal/api/config.go index 9f1999c..32f5b16 100644 --- a/backend/internal/api/config.go +++ b/backend/internal/api/config.go @@ -86,3 +86,19 @@ func savePrometheusHandler(c *gin.Context) { c.Redirect(http.StatusFound, c.Request.Referer()) } + +func saveClickhouseHandler(c *gin.Context) { + + conf.AppConfig.ClickhouseAddr = c.PostForm("addr") + conf.AppConfig.ClickhouseUser = c.PostForm("user") + conf.AppConfig.ClickhousePassword = c.PostForm("password") + conf.AppConfig.ClickhouseDB = c.PostForm("db") + conf.AppConfig.ClickhouseTable = c.PostForm("table") + + enable := c.PostForm("enable") + conf.AppConfig.ClickhouseEnable = enable == "on" + + conf.Write(conf.AppConfig) + + c.Redirect(http.StatusFound, c.Request.Referer()) +} diff --git a/backend/internal/api/routes.go b/backend/internal/api/routes.go index f27223a..19ee0b2 100644 --- a/backend/internal/api/routes.go +++ b/backend/internal/api/routes.go @@ -35,6 +35,7 @@ func Routes(router *gin.Engine) { r0.POST("/config_settings/", saveSettingsHandler) // config.go r0.POST("/config_influx/", saveInfluxHandler) // config.go r0.POST("/config_prometheus/", savePrometheusHandler) // config.go + r0.POST("/config_clickhouse/", saveClickhouseHandler) // config.go } router.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerFiles.Handler)) diff --git a/backend/internal/clickhouse/clickhouse.go b/backend/internal/clickhouse/clickhouse.go new file mode 100644 index 0000000..86ef264 --- /dev/null +++ b/backend/internal/clickhouse/clickhouse.go @@ -0,0 +1,73 @@ +package clickhouse + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "time" + + "github.com/aceberg/WatchYourLAN/internal/models" +) + +// Global HTTP Client to enable Keep-Alive Connection-Reuse +var httpClient = &http.Client{ + Timeout: 10 * time.Second, +} + +// Add - write data to ClickHouse +func Add(appConfig models.Conf, oneHist models.Host) { + if appConfig.ClickhouseAddr == "" { + return + } + + db := appConfig.ClickhouseDB + if db == "" { + db = "default" + } + + table := appConfig.ClickhouseTable + if table == "" { + table = "watch_your_lan" + } + + insertSQL := fmt.Sprintf("INSERT INTO %s.%s (ts, known, mac, iface, ip, name) FORMAT JSONEachRow", db, table) + jsonRow := map[string]any{ + "ts": time.Now().UTC().Format("2006-01-02 15:04:05"), + "known": oneHist.Known, + "mac": oneHist.Mac, + "iface": oneHist.Iface, + "ip": oneHist.IP, + "name": oneHist.Name, + } + jsonData, err := json.Marshal(jsonRow) + if err != nil { + slog.Error("ClickHouse JSON error", "err", err) + return + } + + reqURL := fmt.Sprintf("%s/?query=%s", appConfig.ClickhouseAddr, url.QueryEscape(insertSQL)) + req, err := http.NewRequest(http.MethodPost, reqURL, bytes.NewBuffer(jsonData)) + if err != nil { + slog.Error("ClickHouse Request error", "err", err) + return + } + + req.Header.Set("X-ClickHouse-User", appConfig.ClickhouseUser) + req.Header.Set("X-ClickHouse-Key", appConfig.ClickhousePassword) + + res, err := httpClient.Do(req) + if err != nil { + slog.Error("ClickHouse network error", "err", err) + return + } + defer res.Body.Close() + + body, _ := io.ReadAll(res.Body) + if res.StatusCode != http.StatusOK { + slog.Error("ClickHouse HTTP error", "status", res.StatusCode, "body", string(body)) + } +} diff --git a/backend/internal/conf/read.go b/backend/internal/conf/read.go index 6f5d0ae..c14d5a3 100644 --- a/backend/internal/conf/read.go +++ b/backend/internal/conf/read.go @@ -31,6 +31,8 @@ func read(path string) (config models.Conf) { viper.SetDefault("PROMETHEUS_ENABLE", false) + viper.SetDefault("CLICKHOUSE_ENABLE", false) + viper.SetConfigFile(path) viper.SetConfigType("yaml") err := viper.ReadInConfig() @@ -63,6 +65,13 @@ func read(path string) (config models.Conf) { config.PrometheusEnable = viper.GetBool("PROMETHEUS_ENABLE") + config.ClickhouseEnable = viper.GetBool("CLICKHOUSE_ENABLE") + config.ClickhouseAddr, _ = viper.Get("CLICKHOUSE_ADDR").(string) + config.ClickhouseUser, _ = viper.Get("CLICKHOUSE_USER").(string) + config.ClickhousePassword, _ = viper.Get("CLICKHOUSE_PASSWORD").(string) + config.ClickhouseDB, _ = viper.Get("CLICKHOUSE_DB").(string) + config.ClickhouseTable, _ = viper.Get("CLICKHOUSE_TABLE").(string) + joined := viper.Get("ARP_STRS_JOINED").(string) // slog.Info("ARP_STRS_JOINED: " + joined) diff --git a/backend/internal/conf/write.go b/backend/internal/conf/write.go index 201ecb9..62871ff 100644 --- a/backend/internal/conf/write.go +++ b/backend/internal/conf/write.go @@ -43,6 +43,13 @@ func Write(config models.Conf) { viper.Set("PROMETHEUS_ENABLE", config.PrometheusEnable) + viper.Set("clickhouse_enable", config.ClickhouseEnable) + viper.Set("clickhouse_addr", config.ClickhouseAddr) + viper.Set("clickhouse_user", config.ClickhouseUser) + viper.Set("clickhouse_password", config.ClickhousePassword) + viper.Set("clickhouse_db", config.ClickhouseDB) + viper.Set("clickhouse_table", config.ClickhouseTable) + err := viper.WriteConfig() check.IfError(err) } diff --git a/backend/internal/models/models.go b/backend/internal/models/models.go index 7fcfd16..c8022fc 100644 --- a/backend/internal/models/models.go +++ b/backend/internal/models/models.go @@ -30,6 +30,13 @@ type Conf struct { InfluxSkipTLS bool // Prometheus PrometheusEnable bool + // Clickhouse + ClickhouseEnable bool + ClickhouseAddr string + ClickhouseUser string + ClickhousePassword string + ClickhouseDB string + ClickhouseTable string } // Host - one host diff --git a/backend/internal/routines/scan-routine.go b/backend/internal/routines/scan-routine.go index 2470a01..ce86033 100644 --- a/backend/internal/routines/scan-routine.go +++ b/backend/internal/routines/scan-routine.go @@ -5,6 +5,7 @@ import ( "github.com/aceberg/WatchYourLAN/internal/arp" "github.com/aceberg/WatchYourLAN/internal/check" + "github.com/aceberg/WatchYourLAN/internal/clickhouse" "github.com/aceberg/WatchYourLAN/internal/conf" "github.com/aceberg/WatchYourLAN/internal/gdb" "github.com/aceberg/WatchYourLAN/internal/influx" @@ -79,6 +80,9 @@ func compareHosts(foundHostsMap map[string]models.Host) { if conf.AppConfig.PrometheusEnable { prometheus.Add(aHost) } + if conf.AppConfig.ClickhouseEnable { + clickhouse.Add(conf.AppConfig, aHost) + } } for _, fHost := range foundHostsMap { diff --git a/frontend/src/components/Config/Clickhouse.tsx b/frontend/src/components/Config/Clickhouse.tsx new file mode 100644 index 0000000..da0320a --- /dev/null +++ b/frontend/src/components/Config/Clickhouse.tsx @@ -0,0 +1,54 @@ +import { apiPath } from "../../functions/api" +import { appConfig } from "../../functions/exports" + +function Clickhouse() { + + return ( +
+
ClickHouse config
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Enable +
+ {appConfig().ClickhouseEnable + ? + : + } +
+
Address
User
Password
Database
Table
+
+
+
+ ) +} + +export default Clickhouse diff --git a/frontend/src/functions/exports.ts b/frontend/src/functions/exports.ts index 2cb1ea4..1f78305 100644 --- a/frontend/src/functions/exports.ts +++ b/frontend/src/functions/exports.ts @@ -39,6 +39,13 @@ export interface Conf { InfluxSkipTLS: boolean; // Prometheus PrometheusEnable: boolean; + // Clickhouse + ClickhouseEnable: boolean; + ClickhouseAddr: string; + ClickhouseUser: string; + ClickhousePassword: string; + ClickhouseDB: string; + ClickhouseTable: string; }; export const emptyHost:Host = { @@ -77,6 +84,12 @@ export const emptyConf:Conf = { InfluxBucket: "", InfluxSkipTLS: false, PrometheusEnable: false, + ClickhouseEnable: false, + ClickhouseAddr: "", + ClickhouseUser: "", + ClickhousePassword: "", + ClickhouseDB: "", + ClickhouseTable: "", }; export const [allHosts, setAllHosts] = createStore([]); diff --git a/frontend/src/pages/Config.tsx b/frontend/src/pages/Config.tsx index b5471ef..06efa2d 100644 --- a/frontend/src/pages/Config.tsx +++ b/frontend/src/pages/Config.tsx @@ -1,5 +1,6 @@ import About from "../components/Config/About" import Basic from "../components/Config/Basic" +import Clickhouse from "../components/Config/Clickhouse" import Donate from "../components/Config/Donate" import Influx from "../components/Config/Influx" import Prometheus from "../components/Config/Prometheus" @@ -27,6 +28,9 @@ function Config() {
+
+ +