From 3ca46c9ea2f36ebfcae1617b0a8891291a56a35d Mon Sep 17 00:00:00 2001 From: Oleh Astappiev Date: Wed, 16 Oct 2024 15:57:59 +0200 Subject: [PATCH] first commit --- .assets/icon.png | Bin 0 -> 3817 bytes .github/workflows/go-cross.yml | 47 +++++++++ .github/workflows/main.yml | 72 ++++++++++++++ .gitignore | 6 ++ .golangci.yml | 75 +++++++++++++++ .traefik.yml | 17 ++++ LICENSE | 21 ++++ Makefile | 20 ++++ demo/config.yml | 10 ++ demo/docker-compose.yml | 70 ++++++++++++++ demo/traefik.yml | 30 ++++++ forwarding.go | 106 ++++++++++++++++++++ go.mod | 3 + readme.md | 90 +++++++++++++++++ umami.go | 171 +++++++++++++++++++++++++++++++++ umami_send.go | 72 ++++++++++++++ umami_token.go | 24 +++++ umami_utils.go | 72 ++++++++++++++ umami_websites.go | 52 ++++++++++ 19 files changed, 958 insertions(+) create mode 100644 .assets/icon.png create mode 100644 .github/workflows/go-cross.yml create mode 100644 .github/workflows/main.yml create mode 100644 .gitignore create mode 100644 .golangci.yml create mode 100644 .traefik.yml create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 demo/config.yml create mode 100644 demo/docker-compose.yml create mode 100644 demo/traefik.yml create mode 100644 forwarding.go create mode 100644 go.mod create mode 100644 readme.md create mode 100644 umami.go create mode 100644 umami_send.go create mode 100644 umami_token.go create mode 100644 umami_utils.go create mode 100644 umami_websites.go diff --git a/.assets/icon.png b/.assets/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..cd5f556f33139839833d9f1de676f19f47d4e965 GIT binary patch literal 3817 zcmcIn_dgp>AD6_e-5Ot`MvF>SRP7p#RZ>bzC@MvL?LAsWRFtYxgx1~~QM6WwO%z2F z)ZUfaJN7J|JU=~u!Sln7_xtX1_j=#w?sIqV9~tSfUgEz*Lqo#~)7LSfmRYW<(zz|uunr*-%c0J(Z^UExU(V~g%z1i`=(vT^rkj5& z`(R?q@+~q>)6k$C8^@+Bk$7#u-!xpipdgDLJSLlq_&H!v*`1N7tl1g7-FF`VzBjLB%5dmfB1l76qe}yq&rTlmOmpKF!-MxpcC09|Lgxk41Ux4&9PN}w0;2*ufl#RC-r&tsJZuM&(=N~qB= zbppgz_>s9xpo@zO%m(^XM=+8PdVR}aF{i~OqU9;m-^$~^R`DKsVK2$jwld}8e@&yI zfR&b`+$PRiz!BG@joPz6guUJK3nfyT!+<-xM_&2+oRnwC3}$;-~-?LO{We6)i&vYq3qUk4Y_DZk>ff>3(vdl z?7+`R-XR7v0)@tdz)X;zp?fQ`((<~w<19v;puSjH{KeJHjZeXqtL+`TQx{qkZe}KH zOQ5N*uU8J-5zETT64XGZe%6cUX;>Lw+^wu*In}HX&}XK@1cZm_F1pKB-3yh}Aa}+K z_YD=9rpddQESlQb^mXvsWHQ~?m@OAn_C}qZ9^#z(gkEk;bo6GajyDA>6&hu&8V9!6 zX>1pa3a2I9R*hZj`Ca z*|hUhs(+xs;PYB(1EjdLbknbgz(h_k$ppuID=6p~_bEXkZ8Q1!rH{x|hdaxAOjIqu zzmXkvK$)?rgw79)kMn(dS_4T}@pF`SB&9afFR;qpNwK&T^%R7nbF|XaWH7b2x33D? zUtJ!nu>Y-<&@+y6`aCi+vOMzCE4H|}_~i1ZC6an_`(xUHOx`5I~8;8m1+B z>Hn4o2|lZ3(N`gLM7M2AlSiyo6%`a5HXrq+$#p(-V7lWot1~e0`t2x+%kgoO$fsdjo># zo8-XUal9$=(N{!6vYt?2JHmlUDdc326yI^QtkV#q)H-oUUvZkS}Uksboy^NK?;l=FyOSlI3(MSHa>Z>xfWFX#ardMHf`8d!0 ze!S2_H#ezKo6wL_#GG8$56sjJFjk&^UXg6oZu zc_|_TqxRQE^U+SOVi=Wktif0BY(tQ zB}Q{gxi`Sm%)sGXuFyzeXu8K-6zBCf6(zX?Eg-V4Gpz72C4{Ops$sbl{s}MisKO)cF*`Y zKXHI|%yvCn*!h&bMfx;O-Lbopo6^r3l}8QVmSwai*+%f79}}j>6@P~uPS;^I^_e>E z85*QOsy9r$p0#p(_yxRkZ!)%!dS*$rjYkT>2k!g=0v-*>ZiK=auL*S4beG~ZsT{C5 zMUtXxZxIYh`0?zSPP&CC-8^n_zq`q*$~j@uRjkAk=jh;gZiQ^6nq4u0RvxHR*Pz5n z?J_1m|BWi8aS%`RaO_BHXBHQ^J|~QI7g*}cq2T$uSAGXmsy&1pOBpZAd_q}gZ=B5~ zB_*ZxOTm)6Ki+WKgjv#V&s#|X>eSaFC%u-wGvj(+^2YN(sx-3C11EI`F^u9OIYhkX zmCGO94%Ww61Qc&gF3J+%HQd~g2$Z2L1$=>~#&qO}`(t+;C`}2-Jv`!cfjd_#FTT1En}7Qc9?BhAaj{tGtifPt6Zh$U>xr;xnmi5 zKXL5}p=GgR=9J({mLdeRt)8d|r&t)Lk8th*+Pz;vU4v;=(8VS7=n zsaF=OrF$u8=Uv?7$Gv0q6sdw*wyqmjYi_KVGFz_{%Z>t=o}b5xW#eA!GahQ{G(P^{ zyWsl^D3?A{G!gVN*D}&X>>mioWUX8}A>W&iBTHRO83IZ?D(ut=^5sM0*d`PGRhNoI zQvfO(<#J`Ne<2-wXm|6w!{r33jWQ}mK#rd{ea)wEWt4!hyhnXrcXn=^MH zmc`cK+Vuj;cGuH*ps;NL6~BfO^q$Jb#1Gn2>DQ~&-ok~Q=rvU10=1fR1`+VPiVpd& zp1?r|d#X13&JEa!vU;R70|sOq?+z8Woox!LCofS2HWm z)<3oZxa<#=M!1#4-w&btw?wEl3Fxui|D_D&51iQ@!bdFqQ$DTNrjY_fP%1o_hye(V zcds>a0G5TsTit^;$F5^e&^Q{IKzEz(3Kn<8np|H?e8OD zY+a`WHR(GkK{G}>@Xullw7AzJIn*&8EM6r8W^~8I)I};}+~874vr^$R4^P(La{zsT zsgwpraD9^K*K$5B=xsOWvjWOBj5C{r1KtyoM^#1hswaSRzzad}$TI|A0*e%)1Q0ED z`q8T&j`NronFi3n%co3RE@Y=><^+vE8s}|aNikI)0HyPm9alwkCh2Xtft}r(Rk85QPT4v7|X=&O7gv0nLD36Ise> zU2-YF=-l89j8|}}_Up%jaea}$pRk;<3`nzek0N(T3G?#q~WT^sJ^oV z@23Ci6P%`AS7UPoGs49Y1Y`J(rj+)4GC;H=5s$KsL<#iBq}BG*A^TYtu&xBLSU~HP zVI%)h97XFV^xcZr*ByjD7^5JPx?t!aHgXc?g?1o3GJ3cSrQy)I>am(wz$c2i-^Scmhg!rUsnAZ z-mt#Z_{|I?If8Ghmdk=(a0h{wSr9^dX7pS=(#=zoLTCx}Om8vlvjx-5BMV6lm+#Rd z!5eKU>Y0G^UfQ{uFyT{(B$x}$9xQw$XRGG@j_;aLN!y~{9~0a_m(dKJcq}k+nr|06 zBEL(l|Jm4}$~r+RjI z=Zt#Q{(uW6|MVJp57`);c#z#Phone{Pp@F=J^4e{`dAQzK_WHIyXn}Ph6P+6JLt%G z;^9}fhh^#zBJZPl@9S3r9?vf>F6U{#8!NX9|A+_5wrAQ!W?ZN 0 { + clientIP = strings.Join(values, ", ") + ", " + clientIP + } + dst.Set(xForwardedFor, clientIP) + } + + xfm := req.Header.Get(xForwardedMethod) + switch { + case xfm != "": + dst.Set(xForwardedMethod, xfm) + case req.Method != "": + dst.Set(xForwardedMethod, req.Method) + default: + dst.Del(xForwardedMethod) + } + + xfp := req.Header.Get(xForwardedProto) + switch { + case xfp != "": + dst.Set(xForwardedProto, xfp) + case req.TLS != nil: + dst.Set(xForwardedProto, "https") + default: + dst.Set(xForwardedProto, "http") + } + + if xfp := req.Header.Get(xForwardedPort); xfp != "" { + dst.Set(xForwardedPort, xfp) + } + + xfh := req.Header.Get(xForwardedHost) + switch { + case xfh != "": + dst.Set(xForwardedHost, xfh) + case req.Host != "": + dst.Set(xForwardedHost, req.Host) + default: + dst.Del(xForwardedHost) + } + + xfu := req.Header.Get(xForwardedURI) + switch { + case xfu != "": + dst.Set(xForwardedURI, xfu) + case req.URL.RequestURI() != "": + dst.Set(xForwardedURI, req.URL.RequestURI()) + default: + dst.Del(xForwardedURI) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c3fb628 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/astappiev/traefik-umami-feeder + +go 1.19 diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..d9c3da0 --- /dev/null +++ b/readme.md @@ -0,0 +1,90 @@ +# Umami Feeder for Traefik + +This plugin for enables [Traefik Reverse Proxy](https://traefik.io/traefik/) to feed [Umami Analytics](https://umami.is) +with tracking events. + +It was created as an alternative to [traefik-umami-plugin](https://github.com/1cedsoda/traefik-umami-plugin) and +inspired by idea of [Plausible Feeder Traefik Plugin](https://github.com/safing/plausiblefeeder). + +## Features + +- [X] Super easy to setup, one middleware for all websites +- [X] Server Side Tracking, no need to add JavaScript to your websites +- [X] Fast and private analytics + +## Installation + +To [add this plugin to traefik](https://plugins.traefik.io/install) reference this repository as a plugin in the static +config. +The version references a git tag. + +```yaml +experimental: + plugins: + umami-feeder: + moduleName: github.com/astappiev/traefik-umami-feeder + version: v1.0.0 # replace with latest version available +``` + +```toml +[experimental.plugins.umami-feeder] + moduleName = "github.com/astappiev/traefik-umami-feeder" + version = "v1.0.0" # replace with latest version available +``` + +With the plugin installed, you can configure a middleware in a dynamic configuration such as a `config.yml` or docker +labels. + +```yaml +http: + middlewares: + my-umami-middleware: + plugin: + umami-feeder: + umamiHost: "http://umami:3000" + websites: + "example.com": "d4617504-241c-4797-8eab-5939b367b3ad" +``` + +```toml +[http.middlewares] + [http.middlewares.my-umami-middleware.plugin.umami-feeder] + umamiHost = "umami:3000" + + [http.middlewares.my-umami-middleware.plugin.umami-feeder.websites] + "example.com" = "d4617504-241c-4797-8eab-5939b367b3ad" +``` + +You have an option to give a list of domains to track (and their website IDs on Umami). \ +Or, you can give a token and the list will be fetched from Umami. For this, you need either [retrieve the token yourself](https://umami.is/docs/api/authentication), or use +username/password instead. + +After that, you need to add the middleware to a [router](https://doc.traefik.io/traefik/routing/routers/#middlewares_1). +Remember to reference the +correct [provider namespace](https://doc.traefik.io/traefik/providers/overview/#provider-namespace). + +E.g. as Docker labels: +```yaml +- "traefik.http.routers.whoami.middlewares=my-umami-middleware@file" +``` + +Or, for all routers in a static configuration: +```yaml +entryPoints: + web: + http: + middlewares: + - my-umami-middleware@file +``` + +## Configuration + +| key | default | type | description | +|---------------------|---------|----------|-------------------------------------------------------------------------------------------------------------| +| `umamiHost` | - | `string` | Umami server url, reachable from within traefik (container), e.g. `http://umami:3000` | +| `umamiToken` | - | `string` | An API Token, used to automatize work with websites, not needed if you provide `websites` | +| `umamiUsername` | - | `string` | An alternative to `umamiToken`, you can provide an username and password | +| `umamiPassword` | - | `string` | Only in combination with `umamiUsername` | +| `websites` | - | `map` | A map of hostnames and their associated Umami IDs. Can also be used to override or extend fetched websites | +| `createNewWebsites` | false | `bool` | If set to `true`, will try to create a new website on Umami, if domain not found there | +| `debug` | false | `bool` | Something doesn't work? Set to `true` to see more logs (plugins doesn't have access to Traefik's log level) | diff --git a/umami.go b/umami.go new file mode 100644 index 0000000..2b825f1 --- /dev/null +++ b/umami.go @@ -0,0 +1,171 @@ +package traefik_umami_feeder + +import ( + "context" + "fmt" + "log" + "net/http" + "os" + "time" +) + +// Config the plugin configuration. +type Config struct { + UmamiHost string `json:"umamiHost"` + // it is optional, but either UmamiToken or Websites should be set + UmamiToken string `json:"umamiToken"` + // as an alternative to UmamiToken, you can set UmamiUsername and UmamiPassword to authenticate + UmamiUsername string `json:"umamiUsername"` + UmamiPassword string `json:"umamiPassword"` + // if both UmamiToken and Websites are set, Websites will be used to override the websites in the API + Websites map[string]string `json:"websites"` + // if createNewWebsites is set to true, the plugin will create new websites in the API, UmamiToken is required + CreateNewWebsites bool `json:"createNewWebsites"` + Debug bool `json:"debug"` +} + +// CreateConfig creates the default plugin configuration. +func CreateConfig() *Config { + return &Config{ + UmamiHost: "", + UmamiToken: "", + UmamiUsername: "", + UmamiPassword: "", + Websites: map[string]string{}, + CreateNewWebsites: false, + Debug: false, + } +} + +// UmamiFeeder a UmamiFeeder plugin. +type UmamiFeeder struct { + next http.Handler + name string + debug bool + logHandler *log.Logger + + UmamiHost string + UmamiToken string + Websites map[string]string + CreateNewWebsites bool +} + +// New created a new Demo plugin. +func New(ctx context.Context, next http.Handler, config *Config, name string) (http.Handler, error) { + // construct + h := &UmamiFeeder{ + next: next, + name: name, + debug: config.Debug, + logHandler: log.New(os.Stdout, "", 0), + + UmamiHost: config.UmamiHost, + UmamiToken: config.UmamiToken, + Websites: config.Websites, + CreateNewWebsites: config.CreateNewWebsites, + } + + if config.UmamiUsername != "" && config.UmamiPassword != "" { + token, err := getToken(h.UmamiHost, config.UmamiUsername, config.UmamiPassword) + if err != nil { + return nil, fmt.Errorf("failed to get token: %w", err) + } + if token == "" { + return nil, fmt.Errorf("retrieved token is empty") + } + h.trace("token received %s", token) + h.UmamiToken = token + } + + if h.UmamiHost == "" { + return nil, fmt.Errorf("`umamiHost` is not set") + } + if h.UmamiToken == "" && len(h.Websites) == 0 { + return nil, fmt.Errorf("either `umamiToken` or `websites` should be set") + } + if h.UmamiToken == "" && h.CreateNewWebsites { + return nil, fmt.Errorf("`umamiToken` is required to create new websites") + } + + if h.UmamiToken != "" { + websites, err := fetchWebsites(h.UmamiHost, h.UmamiToken) + if err != nil { + return nil, fmt.Errorf("failed to fetch websites: %w", err) + } + + for _, website := range *websites { + if _, ok := h.Websites[website.Domain]; ok { + continue + } + + h.Websites[website.Domain] = website.ID + h.trace("fetched websiteId for: %s", website.Domain) + } + h.log("websites fetched") + } + + return h, nil +} + +func (h *UmamiFeeder) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + if h.shouldBeTracked(req) { + go h.trackRequest(req) + } else { + h.trace("Tracking skipped %v", req.URL) + } + + h.next.ServeHTTP(rw, req) +} + +func (h *UmamiFeeder) shouldBeTracked(req *http.Request) bool { + if h.CreateNewWebsites { + return true + } + + hostname := parseDomainFromHost(req.Host) + if _, ok := h.Websites[hostname]; ok { + return true + } + + return false +} + +func (h *UmamiFeeder) trackRequest(req *http.Request) { + hostname := parseDomainFromHost(req.Host) + websiteId, ok := h.Websites[hostname] + if !ok { + website, err := createWebsite(h.UmamiHost, h.UmamiToken, hostname) + if err != nil { + h.log("failed to create website: " + err.Error()) + return + } + + h.Websites[website.Domain] = website.ID + websiteId = website.ID + h.trace("created website for: %s", website.Domain) + } + + sendBody, sendHeaders := buildSendBody(req, websiteId) + h.trace("sending tracking request %s with body %v %v", req.URL, sendBody, sendHeaders) + + _, err := sendRequest(h.UmamiHost+"/api/send", sendBody, sendHeaders) + if err != nil { + h.trace("failed to send tracking: " + err.Error()) + return + } +} + +func (h *UmamiFeeder) log(message string) { + if h.logHandler != nil { + time := time.Now().Format("2006-01-02T15:04:05Z") + h.logHandler.Printf("time=\"%s\" level=info msg=\"[traefik-umami-feeder] %s\"", time, message) + } +} + +// Arguments are handled in the manner of [fmt.Printf]. +func (h *UmamiFeeder) trace(format string, v ...any) { + if h.logHandler != nil && h.debug { + time := time.Now().Format("2006-01-02T15:04:05Z") + h.logHandler.Printf("time=\"%s\" level=trace msg=\"[traefik-umami-feeder] %s\"", time, fmt.Sprintf(format, v...)) + } +} diff --git a/umami_send.go b/umami_send.go new file mode 100644 index 0000000..8a3fb73 --- /dev/null +++ b/umami_send.go @@ -0,0 +1,72 @@ +package traefik_umami_feeder + +import ( + "net/http" + "regexp" + "strings" +) + +// Copied from https://github.com/1cedsoda/traefik-umami-plugin/blob/master/umami_tracking.go +// Licensed as Apache-2.0 license + +type SendPayload struct { + Website string `json:"website"` + Hostname string `json:"hostname"` + Language string `json:"language"` + Url string `json:"url"` + Referer string `json:"referer"` + Name string `json:"name"` + Data map[string]interface{} `json:"data"` +} + +type SendBody struct { + Payload SendPayload `json:"payload"` + Type string `json:"type"` +} + +func buildPayload(req *http.Request, websiteId string) SendPayload { + return SendPayload{ + Website: websiteId, + Hostname: parseDomainFromHost(req.Host), + Language: parseAcceptLanguage(req.Header.Get("Accept-Language")), + Url: req.URL.String(), + Referer: req.Referer(), + Name: "traefik", + Data: map[string]interface{}{}, + } +} + +// opts the port from the host. +func parseDomainFromHost(host string) string { + // check if the host has a port + if strings.Contains(host, ":") { + host = strings.Split(host, ":")[0] + } + return host +} + +const parseAcceptLanguagePattern = `([a-zA-Z\-]+)(?:;q=\d\.\d)?(?:,\s)?` + +var parseAcceptLanguageRegexp = regexp.MustCompile(parseAcceptLanguagePattern) + +func parseAcceptLanguage(acceptLanguage string) string { + matches := parseAcceptLanguageRegexp.FindAllStringSubmatch(acceptLanguage, -1) + if len(matches) == 0 { + return "" + } + return matches[0][1] +} + +func buildSendBody(clientReq *http.Request, websiteId string) (SendBody, http.Header) { + body := SendBody{ + Payload: buildPayload(clientReq, websiteId), + Type: "event", + } + + var headers = make(http.Header) + headers.Set("Content-Type", "application/json") + copyHeaders(headers, clientReq.Header) + removeHeaders(headers, hopHeaders...) + writeXForwardedHeaders(headers, clientReq) + return body, headers +} diff --git a/umami_token.go b/umami_token.go new file mode 100644 index 0000000..910dd25 --- /dev/null +++ b/umami_token.go @@ -0,0 +1,24 @@ +package traefik_umami_feeder + +type Auth struct { + Username string `json:"username"` + Password string `json:"password"` +} + +type AuthResponse struct { + Token string `json:"token"` +} + +func getToken(umamiHost string, umamiUsername string, umamiPassword string) (string, error) { + var result AuthResponse + err := sendRequestAndParse(umamiHost+"/api/auth/login", Auth{ + Username: umamiUsername, + Password: umamiPassword, + }, nil, &result) + + if err != nil { + return "", err + } + + return result.Token, nil +} diff --git a/umami_utils.go b/umami_utils.go new file mode 100644 index 0000000..1af6936 --- /dev/null +++ b/umami_utils.go @@ -0,0 +1,72 @@ +package traefik_umami_feeder + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" +) + +func sendRequest(url string, body interface{}, headers http.Header) (*http.Response, error) { + var req *http.Request + var err error + + if body != nil { + bodyJson, err := json.Marshal(body) + if err != nil { + return nil, err + } + + req, err = http.NewRequestWithContext(context.Background(), http.MethodPost, url, bytes.NewReader(bodyJson)) + } else { + req, err = http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + } + + if err != nil { + return nil, err + } + + if headers != nil { + req.Header = headers + } + + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + + client := &http.Client{} + response, err := client.Do(req) + if err != nil { + return nil, err + } + + status := response.StatusCode + if status < 200 || status >= 300 { + return nil, fmt.Errorf("request failed with status %d", status) + } + + return response, nil +} + +func sendRequestAndParse(url string, body interface{}, headers http.Header, value interface{}) error { + resp, err := sendRequest(url, body, headers) + + if err != nil { + return err + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + err = json.Unmarshal(respBody, &value) + if err != nil { + return err + } + + return nil +} diff --git a/umami_websites.go b/umami_websites.go new file mode 100644 index 0000000..79a1207 --- /dev/null +++ b/umami_websites.go @@ -0,0 +1,52 @@ +package traefik_umami_feeder + +import ( + "net/http" + "time" +) + +type WebsitesResponse struct { + Data []Website `json:"data"` + Count int `json:"count"` + Page int `json:"page"` + PageSize int `json:"pageSize"` + OrderBy string `json:"orderBy"` +} + +type Website struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + Domain string `json:"domain"` + CreatedAt time.Time `json:"createdAt,omitempty"` +} + +func createWebsite(umamiHost string, umamiToken string, websiteDomain string) (*Website, error) { + var headers = make(http.Header) + headers.Set("Authorization", "Bearer "+umamiToken) + + var result Website + err := sendRequestAndParse(umamiHost+"/api/websites", Website{ + Name: websiteDomain, + Domain: websiteDomain, + }, headers, &result) + + if err != nil { + return nil, err + } + + return &result, nil +} + +func fetchWebsites(umamiHost string, umamiToken string) (*[]Website, error) { + var headers = make(http.Header) + headers.Set("Authorization", "Bearer "+umamiToken) + + var result WebsitesResponse + err := sendRequestAndParse(umamiHost+"/api/websites?pageSize=200", nil, headers, &result) + + if err != nil { + return nil, err + } + + return &result.Data, nil +}