feat: add ignoreURLs and tests for all ignore* options

demo/docker-compose.yml

@@ -28,6 +28,10 @@ services:
 
   whoami:
     image: containous/whoami
+    ports:
+      - target: 80
+        published: 80
+        protocol: tcp
     labels:
       - "traefik.enable=true"
       - "traefik.http.services.whoami.loadbalancer.server.port=80"

go.mod

@@ -1,3 +1,3 @@
 module github.com/astappiev/traefik-umami-feeder
 
-go 1.19
+go 1.21

readme.md

@@ -82,15 +82,20 @@ entryPoints:
 
 ## Configuration
 
-| key                 | default | type       | description                                                                                                                                 |
+| key                 | default     | type       | description                                                                                                                                   |
-|---------------------|---------|------------|---------------------------------------------------------------------------------------------------------------------------------------------|
+|---------------------|-------------|------------|-----------------------------------------------------------------------------------------------------------------------------------------------|
-| `umamiHost`         | -       | `string`   | Umami server url, reachable from within traefik (container), e.g. `http://umami:3000`                                                       |
+| `disabled`          | false       | `bool`     | Set to `true` to disable the plugin                                                                                                           |
-| `umamiToken`        | -       | `string`   | An API Token, used to automatize work with websites, not needed if you provide `websites`                                                   |
+| `debug`             | false       | `bool`     | Something doesn't work? Set to `true` to see more logs (plugins doesn't have access to Traefik's log level)                                   |
-| `umamiUsername`     | -       | `string`   | An alternative to `umamiToken`, you can provide an username and password                                                                    |
+| `umamiHost`         | -           | `string`   | Umami server url, reachable from within traefik (container), e.g. `http://umami:3000`                                                         |
-| `umamiPassword`     | -       | `string`   | Only in combination with `umamiUsername`                                                                                                    |
+| `umamiToken`        | -           | `string`   | An API Token, used to automatize work with websites, not needed if you provide `websites`                                                     |
-| `umamiTeamId`       | -       | `string`   | In order to organize websites, you can use Umami Teams                                                                                      |
+| `umamiUsername`     | -           | `string`   | An alternative to `umamiToken`, you can provide an username and password                                                                      |
-| `websites`          | -       | `map`      | A map of hostnames and their associated Umami IDs. Can also be used to override or extend fetched websites                                  |
+| `umamiPassword`     | -           | `string`   | Only in combination with `umamiUsername`                                                                                                      |
-| `createNewWebsites` | false   | `bool`     | If set to `true`, will try to create a new website on Umami, if domain not found there                                                      |
+| `umamiTeamId`       | -           | `string`   | In order to organize websites, you can use Umami Teams                                                                                        |
-| `trackAllResources` | false   | `bool`     | Defines whether all requests for any resource should be tracked. By default, only requests that are believed to contain content are tracked |
+| `websites`          | -           | `map`      | A map of hostnames and their associated Umami IDs. Can also be used to override or extend fetched websites                                    |
-| `trackExtensions`   |         | `string[]` | Defines an alternative list of file extensions that should be tracked                                                                       |
+| `createNewWebsites` | false       | `bool`     | If set to `true`, will try to create a new website on Umami, if domain not found there                                                        |
-| `debug`             | false   | `bool`     | Something doesn't work? Set to `true` to see more logs (plugins doesn't have access to Traefik's log level)                                 |
+| `trackAllResources` | false       | `bool`     | Defines whether all requests for any resource should be tracked. By default, only requests that are believed to contain content are tracked   |
+| `trackExtensions`   |             | `string[]` | Defines an alternative list of file extensions that should be tracked                                                                         |
+| `ignoreUserAgents`  |             | `string[]` | A list of user agents that should be ignored from tracking, e.g. `["Googlebot", "Uptime-Kuma"]` (matched with `strings.Contains`)             |
+| `ignoreURLs`        |             | `string[]` | A list of URLs that should be ignored from tracking, e.g. `["/health", "https?://[^/]+/health$"]` (matched with `regexp.Compile.MatchString`) |
+| `ignoreIPs`         |             | `string[]` | A list of IPs that should be ignored from tracking, e.g. `["127.0.0.1", "10.0.0.1/16"]` (matched with `netip.ParsePrefix.Contains`)           |
+| `headerIp`          | `X-Real-Ip` | `string`   | The header to use to get the real IP address of the client, in case it's forwarded by a proxy                                                 |

umami.go

@@ -8,6 +8,7 @@ import (
 	"net/netip"
 	"os"
 	"path"
+	"regexp"
 	"strings"
 	"time"
 )
@@ -42,9 +43,11 @@ type Config struct {
 	// TrackExtensions defines an alternative list of file extensions that should be tracked.
 	TrackExtensions []string `json:"trackExtensions"`
 
-	// IgnoreUserAgents is a list of user agents that should be ignored.
+	// IgnoreUserAgents is a list of user agents to ignore.
 	IgnoreUserAgents []string `json:"ignoreUserAgents"`
-	// IgnoreIPs is a list of IPs or CIDRs that should be ignored.
+	// IgnoreURLs is a list of request urls to ignore, each string is converted to RegExp and urls matched against it.
+	IgnoreURLs []string `json:"ignoreURLs"`
+	// IgnoreIPs is a list of IPs or CIDRs to ignore.
 	IgnoreIPs []string `json:"ignoreIPs"`
 	// headerIp Header associated to real IP
 	HeaderIp string `json:"headerIp"`
@@ -69,6 +72,7 @@ func CreateConfig() *Config {
 		TrackExtensions:   []string{},
 
 		IgnoreUserAgents: []string{},
+		IgnoreURLs:       []string{},
 		IgnoreIPs:        []string{},
 		HeaderIp:         "X-Real-Ip",
 	}
@@ -92,6 +96,7 @@ type UmamiFeeder struct {
 	trackExtensions   []string
 
 	ignoreUserAgents []string
+	ignoreRegexps    []regexp.Regexp
 	ignorePrefixes   []netip.Prefix
 	headerIp         string
 }
@@ -116,42 +121,31 @@ func New(ctx context.Context, next http.Handler, config *Config, name string) (h
 		trackExtensions:   config.TrackExtensions,
 
 		ignoreUserAgents: config.IgnoreUserAgents,
+		ignoreRegexps:    []regexp.Regexp{},
 		ignorePrefixes:   []netip.Prefix{},
 		headerIp:         config.HeaderIp,
 	}
 
 	if !h.isDisabled {
-		err := h.verifyConfig(config)
+		err := h.connect(config)
 		if err != nil {
 			h.error(err.Error())
-			h.error("due to the error, the Umami plugin is disabled")
+			h.error("unable to connect to Umami, the plugin is disabled")
 			h.isDisabled = true
 		}
-	}
 
-	if len(config.IgnoreIPs) > 0 {
+		err = h.verifyConfig(config)
-		for _, ignoreIp := range config.IgnoreIPs {
+		if err != nil {
-			network, err := netip.ParsePrefix(ignoreIp)
+			h.error(err.Error())
-			if err != nil {
+			h.error("configuration error, the plugin is disabled")
-				network, err = netip.ParsePrefix(ignoreIp + "/32")
+			h.isDisabled = true
-			}
-
-			if err != nil || !network.IsValid() {
-				if err != nil {
-					h.error(err.Error())
-				}
-				h.error(fmt.Sprintf("invalid ignoreIp given %s, this param accepts only IP addresses or CIRD in a format 10.0.0.1/16", ignoreIp))
-				h.isDisabled = true
-			} else {
-				h.ignorePrefixes = append(h.ignorePrefixes, network)
-			}
 		}
 	}
 
 	return h, nil
 }
 
-func (h *UmamiFeeder) verifyConfig(config *Config) error {
+func (h *UmamiFeeder) connect(config *Config) error {
 	if h.umamiHost == "" {
 		return fmt.Errorf("`umamiHost` is not set")
 	}
@@ -193,28 +187,45 @@ func (h *UmamiFeeder) verifyConfig(config *Config) error {
 	return nil
 }
 
-func (h *UmamiFeeder) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
+func (h *UmamiFeeder) verifyConfig(config *Config) error {
-	if !h.isDisabled {
+	if len(config.IgnoreIPs) > 0 {
-		if h.shouldTrack(req) {
+		for _, ignoreIp := range config.IgnoreIPs {
-			go h.trackRequest(req)
+			network, err := netip.ParsePrefix(ignoreIp)
-		} else {
+			if err != nil {
-			h.debug("ignoring request to %s%s", req.Host, req.URL)
+				network, err = netip.ParsePrefix(ignoreIp + "/32")
+			}
+
+			if err != nil || !network.IsValid() {
+				return fmt.Errorf("invalid ignoreIp given %s: %w", ignoreIp, err)
+			}
+
+			h.ignorePrefixes = append(h.ignorePrefixes, network)
 		}
 	}
 
+	if len(config.IgnoreURLs) > 0 {
+		for _, location := range config.IgnoreURLs {
+			r, err := regexp.Compile(location)
+			if err != nil {
+				return fmt.Errorf("failed to compile ignoreURL %s: %w", location, err)
+			}
+
+			h.ignoreRegexps = append(h.ignoreRegexps, *r)
+		}
+	}
+
+	return nil
+}
+
+func (h *UmamiFeeder) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
+	if !h.isDisabled && h.shouldTrack(req) {
+		go h.trackRequest(req)
+	}
+
 	h.next.ServeHTTP(rw, req)
 }
 
 func (h *UmamiFeeder) shouldTrack(req *http.Request) bool {
-	if len(h.ignoreUserAgents) > 0 {
-		userAgent := req.UserAgent()
-		for _, disabledUserAgent := range h.ignoreUserAgents {
-			if strings.Contains(userAgent, disabledUserAgent) {
-				return false
-			}
-		}
-	}
-
 	if len(h.ignorePrefixes) > 0 {
 		requestIp := req.Header.Get(h.headerIp)
 		if requestIp == "" {
@@ -229,12 +240,34 @@ func (h *UmamiFeeder) shouldTrack(req *http.Request) bool {
 
 		for _, prefix := range h.ignorePrefixes {
 			if prefix.Contains(ip) {
+				h.debug("ignoring IP %s", ip)
+				return false
+			}
+		}
+	}
+
+	if len(h.ignoreUserAgents) > 0 {
+		userAgent := req.UserAgent()
+		for _, disabledUserAgent := range h.ignoreUserAgents {
+			if strings.Contains(userAgent, disabledUserAgent) {
+				h.debug("ignoring user-agent %s", userAgent)
+				return false
+			}
+		}
+	}
+
+	if len(h.ignoreRegexps) > 0 {
+		requestURL := req.URL.String()
+		for _, r := range h.ignoreRegexps {
+			if r.MatchString(requestURL) {
+				h.debug("ignoring location %s", requestURL)
 				return false
 			}
 		}
 	}
 
 	if !h.shouldTrackResource(req.URL.Path) {
+		h.debug("ignoring resource %s", req.URL.Path)
 		return false
 	}
 
@@ -247,6 +280,7 @@ func (h *UmamiFeeder) shouldTrack(req *http.Request) bool {
 		return true
 	}
 
+	h.debug("ignoring domain %s", hostname)
 	return false
 }
 
@@ -269,18 +303,7 @@ func (h *UmamiFeeder) shouldTrackResource(url string) bool {
 
 	// Check if the suffix is regarded to be "content".
 	switch pathExt {
-	case ".htm":
+	case "", ".htm", ".html", ".xhtml", ".jsf", ".md", ".php", ".rss", ".rtf", ".txt", ".xml", ".pdf":
-	case ".html":
-	case ".xhtml":
-	case ".jsf":
-	case ".md":
-	case ".php":
-	case ".rss":
-	case ".rtf":
-	case ".txt":
-	case ".xml":
-	case ".pdf":
-	case "":
 		return true
 	}
 
@@ -314,15 +337,15 @@ func (h *UmamiFeeder) trackRequest(req *http.Request) {
 
 func (h *UmamiFeeder) error(message string) {
 	if h.logHandler != nil {
-		time := time.Now().Format("2006-01-02T15:04:05Z")
+		now := time.Now().Format("2006-01-02T15:04:05Z")
-		h.logHandler.Printf("%s ERR middlewareName=%s error=\"%s\"", time, h.name, message)
+		h.logHandler.Printf("%s ERR middlewareName=%s error=\"%s\"", now, h.name, message)
 	}
 }
 
 // Arguments are handled in the manner of [fmt.Printf].
 func (h *UmamiFeeder) debug(format string, v ...any) {
 	if h.logHandler != nil && h.isDebug {
-		time := time.Now().Format("2006-01-02T15:04:05Z")
+		now := time.Now().Format("2006-01-02T15:04:05Z")
-		h.logHandler.Printf("%s DBG middlewareName=%s msg=\"%s\"", time, h.name, fmt.Sprintf(format, v...))
+		h.logHandler.Printf("%s DBG middlewareName=%s msg=\"%s\"", now, h.name, fmt.Sprintf(format, v...))
 	}
 }

umami_test.go

@@ -0,0 +1,145 @@
+package traefik_umami_feeder
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestTraefikUmamiFeeder(t *testing.T) {
+	t.Skip("requires a running Umami instance, see /demo/docker-compose.yml")
+
+	cfg := CreateConfig()
+	cfg.UmamiHost = "http://localhost:3000"
+	cfg.UmamiUsername = "admin"
+	cfg.UmamiPassword = "umami"
+	cfg.UmamiTeamId = "8e39c6ad-e44a-4d3e-be98-015db2d62d40"
+	cfg.CreateNewWebsites = true
+
+	ctx := context.Background()
+	next := http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {})
+
+	handler, err := New(ctx, next, cfg, "umami-feeder")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	recorder := httptest.NewRecorder()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:80", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	handler.ServeHTTP(recorder, req)
+}
+
+func TestShouldTrackDefault(t *testing.T) {
+	feeder := UmamiFeeder{}
+
+	assertResource(t, feeder, true, "http://localhost")
+	assertResource(t, feeder, true, "http://localhost/about")
+	assertResource(t, feeder, true, "http://localhost/products.html")
+	assertResource(t, feeder, true, "http://localhost/blog.php")
+	assertResource(t, feeder, true, "http://localhost/feed.rss")
+	assertResource(t, feeder, false, "http://localhost/favicon.ico")
+	assertResource(t, feeder, false, "http://localhost/photo.jpg")
+	assertResource(t, feeder, false, "http://localhost/background.png")
+}
+
+func assertResource(t *testing.T, plugin UmamiFeeder, expected bool, url string) {
+	if expected != plugin.shouldTrackResource(url) {
+		t.Fatalf("expected %v for %s", expected, url)
+	}
+}
+
+func TestShouldTrackInvalidIp(t *testing.T) {
+	feeder := UmamiFeeder{}
+	err := feeder.verifyConfig(&Config{
+		IgnoreIPs: []string{"127.0.0.1-127.0.0.10"},
+	})
+
+	if err == nil {
+		t.Fatal("should have failed with invalid IP")
+	}
+}
+
+func TestShouldTrackIps(t *testing.T) {
+	feeder := UmamiFeeder{createNewWebsites: true}
+	err := feeder.verifyConfig(&Config{
+		IgnoreIPs: []string{"127.0.0.1", "10.0.0.1/24"},
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertIgnoreIp(t, feeder, true, "192.168.0.1")
+	assertIgnoreIp(t, feeder, false, "127.0.0.1")
+	assertIgnoreIp(t, feeder, false, "10.0.0.1")
+	assertIgnoreIp(t, feeder, false, "10.0.0.255")
+	assertIgnoreIp(t, feeder, true, "10.0.1.1")
+	assertIgnoreIp(t, feeder, true, "10.10.10.1")
+	assertIgnoreIp(t, feeder, true, "1.1.1.1")
+	assertIgnoreIp(t, feeder, true, "8.8.8.8")
+}
+
+func assertIgnoreIp(t *testing.T, plugin UmamiFeeder, expected bool, clientIp string) {
+	req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://localhost", nil)
+	req.Header.Set(plugin.headerIp, clientIp)
+
+	if expected != plugin.shouldTrack(req) {
+		t.Fatalf("expected %v for %s", expected, clientIp)
+	}
+}
+
+func TestShouldTrackUrls(t *testing.T) {
+	feeder := UmamiFeeder{createNewWebsites: true}
+	err := feeder.verifyConfig(&Config{
+		IgnoreURLs: []string{"https?://[^/]+/health$", "/about"},
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertIgnoreUrl(t, feeder, false, "http://localhost/health")
+	assertIgnoreUrl(t, feeder, true, "http://localhost/user/health")
+	assertIgnoreUrl(t, feeder, true, "http://localhost/healthcheck")
+	assertIgnoreUrl(t, feeder, true, "http://localhost/")
+	assertIgnoreUrl(t, feeder, false, "http://localhost/about")
+	assertIgnoreUrl(t, feeder, false, "http://localhost/aboutus")
+	assertIgnoreUrl(t, feeder, false, "http://localhost/category/about")
+	assertIgnoreUrl(t, feeder, true, "http://localhost/hello-world")
+}
+
+func assertIgnoreUrl(t *testing.T, plugin UmamiFeeder, expected bool, url string) {
+	req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
+
+	if expected != plugin.shouldTrack(req) {
+		t.Fatalf("expected %v for %s", expected, url)
+	}
+}
+
+func TestShouldTrackUserAgents(t *testing.T) {
+	feeder := UmamiFeeder{createNewWebsites: true, ignoreUserAgents: []string{"Googlebot", "Uptime-Kuma"}}
+
+	assertIgnoreUa(t, feeder, true, "Mozilla/5.0 (Windows; Windows NT 6.0; WOW64) Gecko/20100101 Firefox/60.7")
+	assertIgnoreUa(t, feeder, true, "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 10.0; Win64; x64 Trident/6.0)")
+	assertIgnoreUa(t, feeder, true, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
+	assertIgnoreUa(t, feeder, false, "Uptime-Kuma/1.18.5")
+	assertIgnoreUa(t, feeder, false, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4430.212 Safari/537.36 Uptime-Kuma/1.23.1")
+	assertIgnoreUa(t, feeder, true, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
+	assertIgnoreUa(t, feeder, false, "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
+	assertIgnoreUa(t, feeder, false, "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
+}
+
+func assertIgnoreUa(t *testing.T, plugin UmamiFeeder, expected bool, ua string) {
+	req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://localhost/", nil)
+	req.Header.Set("User-Agent", ua)
+
+	if expected != plugin.shouldTrack(req) {
+		t.Fatalf("expected %v for %s", expected, ua)
+	}
+}