diff --git a/demo/docker-compose.yml b/demo/docker-compose.yml index 221af77..536ca95 100644 --- a/demo/docker-compose.yml +++ b/demo/docker-compose.yml @@ -28,6 +28,10 @@ services: whoami: image: containous/whoami + ports: + - target: 80 + published: 80 + protocol: tcp labels: - "traefik.enable=true" - "traefik.http.services.whoami.loadbalancer.server.port=80" diff --git a/go.mod b/go.mod index c3fb628..726d182 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/astappiev/traefik-umami-feeder -go 1.19 +go 1.21 diff --git a/readme.md b/readme.md index fbaa403..172e2c6 100644 --- a/readme.md +++ b/readme.md @@ -82,15 +82,20 @@ entryPoints: ## Configuration -| key | default | type | description | -|---------------------|---------|------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| `umamiHost` | - | `string` | Umami server url, reachable from within traefik (container), e.g. `http://umami:3000` | -| `umamiToken` | - | `string` | An API Token, used to automatize work with websites, not needed if you provide `websites` | -| `umamiUsername` | - | `string` | An alternative to `umamiToken`, you can provide an username and password | -| `umamiPassword` | - | `string` | Only in combination with `umamiUsername` | -| `umamiTeamId` | - | `string` | In order to organize websites, you can use Umami Teams | -| `websites` | - | `map` | A map of hostnames and their associated Umami IDs. Can also be used to override or extend fetched websites | -| `createNewWebsites` | false | `bool` | If set to `true`, will try to create a new website on Umami, if domain not found there | -| `trackAllResources` | false | `bool` | Defines whether all requests for any resource should be tracked. By default, only requests that are believed to contain content are tracked | -| `trackExtensions` | | `string[]` | Defines an alternative list of file extensions that should be tracked | -| `debug` | false | `bool` | Something doesn't work? Set to `true` to see more logs (plugins doesn't have access to Traefik's log level) | +| key | default | type | description | +|---------------------|-------------|------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| `disabled` | false | `bool` | Set to `true` to disable the plugin | +| `debug` | false | `bool` | Something doesn't work? Set to `true` to see more logs (plugins doesn't have access to Traefik's log level) | +| `umamiHost` | - | `string` | Umami server url, reachable from within traefik (container), e.g. `http://umami:3000` | +| `umamiToken` | - | `string` | An API Token, used to automatize work with websites, not needed if you provide `websites` | +| `umamiUsername` | - | `string` | An alternative to `umamiToken`, you can provide an username and password | +| `umamiPassword` | - | `string` | Only in combination with `umamiUsername` | +| `umamiTeamId` | - | `string` | In order to organize websites, you can use Umami Teams | +| `websites` | - | `map` | A map of hostnames and their associated Umami IDs. Can also be used to override or extend fetched websites | +| `createNewWebsites` | false | `bool` | If set to `true`, will try to create a new website on Umami, if domain not found there | +| `trackAllResources` | false | `bool` | Defines whether all requests for any resource should be tracked. By default, only requests that are believed to contain content are tracked | +| `trackExtensions` | | `string[]` | Defines an alternative list of file extensions that should be tracked | +| `ignoreUserAgents` | | `string[]` | A list of user agents that should be ignored from tracking, e.g. `["Googlebot", "Uptime-Kuma"]` (matched with `strings.Contains`) | +| `ignoreURLs` | | `string[]` | A list of URLs that should be ignored from tracking, e.g. `["/health", "https?://[^/]+/health$"]` (matched with `regexp.Compile.MatchString`) | +| `ignoreIPs` | | `string[]` | A list of IPs that should be ignored from tracking, e.g. `["127.0.0.1", "10.0.0.1/16"]` (matched with `netip.ParsePrefix.Contains`) | +| `headerIp` | `X-Real-Ip` | `string` | The header to use to get the real IP address of the client, in case it's forwarded by a proxy | diff --git a/umami.go b/umami.go index 01c0b61..939d823 100644 --- a/umami.go +++ b/umami.go @@ -8,6 +8,7 @@ import ( "net/netip" "os" "path" + "regexp" "strings" "time" ) @@ -42,9 +43,11 @@ type Config struct { // TrackExtensions defines an alternative list of file extensions that should be tracked. TrackExtensions []string `json:"trackExtensions"` - // IgnoreUserAgents is a list of user agents that should be ignored. + // IgnoreUserAgents is a list of user agents to ignore. IgnoreUserAgents []string `json:"ignoreUserAgents"` - // IgnoreIPs is a list of IPs or CIDRs that should be ignored. + // IgnoreURLs is a list of request urls to ignore, each string is converted to RegExp and urls matched against it. + IgnoreURLs []string `json:"ignoreURLs"` + // IgnoreIPs is a list of IPs or CIDRs to ignore. IgnoreIPs []string `json:"ignoreIPs"` // headerIp Header associated to real IP HeaderIp string `json:"headerIp"` @@ -69,6 +72,7 @@ func CreateConfig() *Config { TrackExtensions: []string{}, IgnoreUserAgents: []string{}, + IgnoreURLs: []string{}, IgnoreIPs: []string{}, HeaderIp: "X-Real-Ip", } @@ -92,6 +96,7 @@ type UmamiFeeder struct { trackExtensions []string ignoreUserAgents []string + ignoreRegexps []regexp.Regexp ignorePrefixes []netip.Prefix headerIp string } @@ -116,42 +121,31 @@ func New(ctx context.Context, next http.Handler, config *Config, name string) (h trackExtensions: config.TrackExtensions, ignoreUserAgents: config.IgnoreUserAgents, + ignoreRegexps: []regexp.Regexp{}, ignorePrefixes: []netip.Prefix{}, headerIp: config.HeaderIp, } if !h.isDisabled { - err := h.verifyConfig(config) + err := h.connect(config) if err != nil { h.error(err.Error()) - h.error("due to the error, the Umami plugin is disabled") + h.error("unable to connect to Umami, the plugin is disabled") h.isDisabled = true } - } - if len(config.IgnoreIPs) > 0 { - for _, ignoreIp := range config.IgnoreIPs { - network, err := netip.ParsePrefix(ignoreIp) - if err != nil { - network, err = netip.ParsePrefix(ignoreIp + "/32") - } - - if err != nil || !network.IsValid() { - if err != nil { - h.error(err.Error()) - } - h.error(fmt.Sprintf("invalid ignoreIp given %s, this param accepts only IP addresses or CIRD in a format 10.0.0.1/16", ignoreIp)) - h.isDisabled = true - } else { - h.ignorePrefixes = append(h.ignorePrefixes, network) - } + err = h.verifyConfig(config) + if err != nil { + h.error(err.Error()) + h.error("configuration error, the plugin is disabled") + h.isDisabled = true } } return h, nil } -func (h *UmamiFeeder) verifyConfig(config *Config) error { +func (h *UmamiFeeder) connect(config *Config) error { if h.umamiHost == "" { return fmt.Errorf("`umamiHost` is not set") } @@ -193,28 +187,45 @@ func (h *UmamiFeeder) verifyConfig(config *Config) error { return nil } -func (h *UmamiFeeder) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - if !h.isDisabled { - if h.shouldTrack(req) { - go h.trackRequest(req) - } else { - h.debug("ignoring request to %s%s", req.Host, req.URL) +func (h *UmamiFeeder) verifyConfig(config *Config) error { + if len(config.IgnoreIPs) > 0 { + for _, ignoreIp := range config.IgnoreIPs { + network, err := netip.ParsePrefix(ignoreIp) + if err != nil { + network, err = netip.ParsePrefix(ignoreIp + "/32") + } + + if err != nil || !network.IsValid() { + return fmt.Errorf("invalid ignoreIp given %s: %w", ignoreIp, err) + } + + h.ignorePrefixes = append(h.ignorePrefixes, network) } } + if len(config.IgnoreURLs) > 0 { + for _, location := range config.IgnoreURLs { + r, err := regexp.Compile(location) + if err != nil { + return fmt.Errorf("failed to compile ignoreURL %s: %w", location, err) + } + + h.ignoreRegexps = append(h.ignoreRegexps, *r) + } + } + + return nil +} + +func (h *UmamiFeeder) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + if !h.isDisabled && h.shouldTrack(req) { + go h.trackRequest(req) + } + h.next.ServeHTTP(rw, req) } func (h *UmamiFeeder) shouldTrack(req *http.Request) bool { - if len(h.ignoreUserAgents) > 0 { - userAgent := req.UserAgent() - for _, disabledUserAgent := range h.ignoreUserAgents { - if strings.Contains(userAgent, disabledUserAgent) { - return false - } - } - } - if len(h.ignorePrefixes) > 0 { requestIp := req.Header.Get(h.headerIp) if requestIp == "" { @@ -229,12 +240,34 @@ func (h *UmamiFeeder) shouldTrack(req *http.Request) bool { for _, prefix := range h.ignorePrefixes { if prefix.Contains(ip) { + h.debug("ignoring IP %s", ip) + return false + } + } + } + + if len(h.ignoreUserAgents) > 0 { + userAgent := req.UserAgent() + for _, disabledUserAgent := range h.ignoreUserAgents { + if strings.Contains(userAgent, disabledUserAgent) { + h.debug("ignoring user-agent %s", userAgent) + return false + } + } + } + + if len(h.ignoreRegexps) > 0 { + requestURL := req.URL.String() + for _, r := range h.ignoreRegexps { + if r.MatchString(requestURL) { + h.debug("ignoring location %s", requestURL) return false } } } if !h.shouldTrackResource(req.URL.Path) { + h.debug("ignoring resource %s", req.URL.Path) return false } @@ -247,6 +280,7 @@ func (h *UmamiFeeder) shouldTrack(req *http.Request) bool { return true } + h.debug("ignoring domain %s", hostname) return false } @@ -269,18 +303,7 @@ func (h *UmamiFeeder) shouldTrackResource(url string) bool { // Check if the suffix is regarded to be "content". switch pathExt { - case ".htm": - case ".html": - case ".xhtml": - case ".jsf": - case ".md": - case ".php": - case ".rss": - case ".rtf": - case ".txt": - case ".xml": - case ".pdf": - case "": + case "", ".htm", ".html", ".xhtml", ".jsf", ".md", ".php", ".rss", ".rtf", ".txt", ".xml", ".pdf": return true } @@ -314,15 +337,15 @@ func (h *UmamiFeeder) trackRequest(req *http.Request) { func (h *UmamiFeeder) error(message string) { if h.logHandler != nil { - time := time.Now().Format("2006-01-02T15:04:05Z") - h.logHandler.Printf("%s ERR middlewareName=%s error=\"%s\"", time, h.name, message) + now := time.Now().Format("2006-01-02T15:04:05Z") + h.logHandler.Printf("%s ERR middlewareName=%s error=\"%s\"", now, h.name, message) } } // Arguments are handled in the manner of [fmt.Printf]. func (h *UmamiFeeder) debug(format string, v ...any) { if h.logHandler != nil && h.isDebug { - time := time.Now().Format("2006-01-02T15:04:05Z") - h.logHandler.Printf("%s DBG middlewareName=%s msg=\"%s\"", time, h.name, fmt.Sprintf(format, v...)) + now := time.Now().Format("2006-01-02T15:04:05Z") + h.logHandler.Printf("%s DBG middlewareName=%s msg=\"%s\"", now, h.name, fmt.Sprintf(format, v...)) } } diff --git a/umami_test.go b/umami_test.go new file mode 100644 index 0000000..57c674d --- /dev/null +++ b/umami_test.go @@ -0,0 +1,145 @@ +package traefik_umami_feeder + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +func TestTraefikUmamiFeeder(t *testing.T) { + t.Skip("requires a running Umami instance, see /demo/docker-compose.yml") + + cfg := CreateConfig() + cfg.UmamiHost = "http://localhost:3000" + cfg.UmamiUsername = "admin" + cfg.UmamiPassword = "umami" + cfg.UmamiTeamId = "8e39c6ad-e44a-4d3e-be98-015db2d62d40" + cfg.CreateNewWebsites = true + + ctx := context.Background() + next := http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {}) + + handler, err := New(ctx, next, cfg, "umami-feeder") + if err != nil { + t.Fatal(err) + } + + recorder := httptest.NewRecorder() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:80", nil) + if err != nil { + t.Fatal(err) + } + + handler.ServeHTTP(recorder, req) +} + +func TestShouldTrackDefault(t *testing.T) { + feeder := UmamiFeeder{} + + assertResource(t, feeder, true, "http://localhost") + assertResource(t, feeder, true, "http://localhost/about") + assertResource(t, feeder, true, "http://localhost/products.html") + assertResource(t, feeder, true, "http://localhost/blog.php") + assertResource(t, feeder, true, "http://localhost/feed.rss") + assertResource(t, feeder, false, "http://localhost/favicon.ico") + assertResource(t, feeder, false, "http://localhost/photo.jpg") + assertResource(t, feeder, false, "http://localhost/background.png") +} + +func assertResource(t *testing.T, plugin UmamiFeeder, expected bool, url string) { + if expected != plugin.shouldTrackResource(url) { + t.Fatalf("expected %v for %s", expected, url) + } +} + +func TestShouldTrackInvalidIp(t *testing.T) { + feeder := UmamiFeeder{} + err := feeder.verifyConfig(&Config{ + IgnoreIPs: []string{"127.0.0.1-127.0.0.10"}, + }) + + if err == nil { + t.Fatal("should have failed with invalid IP") + } +} + +func TestShouldTrackIps(t *testing.T) { + feeder := UmamiFeeder{createNewWebsites: true} + err := feeder.verifyConfig(&Config{ + IgnoreIPs: []string{"127.0.0.1", "10.0.0.1/24"}, + }) + + if err != nil { + t.Fatal(err) + } + + assertIgnoreIp(t, feeder, true, "192.168.0.1") + assertIgnoreIp(t, feeder, false, "127.0.0.1") + assertIgnoreIp(t, feeder, false, "10.0.0.1") + assertIgnoreIp(t, feeder, false, "10.0.0.255") + assertIgnoreIp(t, feeder, true, "10.0.1.1") + assertIgnoreIp(t, feeder, true, "10.10.10.1") + assertIgnoreIp(t, feeder, true, "1.1.1.1") + assertIgnoreIp(t, feeder, true, "8.8.8.8") +} + +func assertIgnoreIp(t *testing.T, plugin UmamiFeeder, expected bool, clientIp string) { + req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://localhost", nil) + req.Header.Set(plugin.headerIp, clientIp) + + if expected != plugin.shouldTrack(req) { + t.Fatalf("expected %v for %s", expected, clientIp) + } +} + +func TestShouldTrackUrls(t *testing.T) { + feeder := UmamiFeeder{createNewWebsites: true} + err := feeder.verifyConfig(&Config{ + IgnoreURLs: []string{"https?://[^/]+/health$", "/about"}, + }) + + if err != nil { + t.Fatal(err) + } + + assertIgnoreUrl(t, feeder, false, "http://localhost/health") + assertIgnoreUrl(t, feeder, true, "http://localhost/user/health") + assertIgnoreUrl(t, feeder, true, "http://localhost/healthcheck") + assertIgnoreUrl(t, feeder, true, "http://localhost/") + assertIgnoreUrl(t, feeder, false, "http://localhost/about") + assertIgnoreUrl(t, feeder, false, "http://localhost/aboutus") + assertIgnoreUrl(t, feeder, false, "http://localhost/category/about") + assertIgnoreUrl(t, feeder, true, "http://localhost/hello-world") +} + +func assertIgnoreUrl(t *testing.T, plugin UmamiFeeder, expected bool, url string) { + req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + + if expected != plugin.shouldTrack(req) { + t.Fatalf("expected %v for %s", expected, url) + } +} + +func TestShouldTrackUserAgents(t *testing.T) { + feeder := UmamiFeeder{createNewWebsites: true, ignoreUserAgents: []string{"Googlebot", "Uptime-Kuma"}} + + assertIgnoreUa(t, feeder, true, "Mozilla/5.0 (Windows; Windows NT 6.0; WOW64) Gecko/20100101 Firefox/60.7") + assertIgnoreUa(t, feeder, true, "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 10.0; Win64; x64 Trident/6.0)") + assertIgnoreUa(t, feeder, true, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36") + assertIgnoreUa(t, feeder, false, "Uptime-Kuma/1.18.5") + assertIgnoreUa(t, feeder, false, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4430.212 Safari/537.36 Uptime-Kuma/1.23.1") + assertIgnoreUa(t, feeder, true, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36") + assertIgnoreUa(t, feeder, false, "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)") + assertIgnoreUa(t, feeder, false, "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)") +} + +func assertIgnoreUa(t *testing.T, plugin UmamiFeeder, expected bool, ua string) { + req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://localhost/", nil) + req.Header.Set("User-Agent", ua) + + if expected != plugin.shouldTrack(req) { + t.Fatalf("expected %v for %s", expected, ua) + } +}