feat: add ignoreURLs and tests for all ignore* options
This commit is contained in:
Oleh Astappiev
parent
b6d30d47ac
commit
465ea8b64e
5 changed files with 243 additions and 66 deletions
|
|
@ -28,6 +28,10 @@ services:
|
|||
|
||||
whoami:
|
||||
image: containous/whoami
|
||||
ports:
|
||||
- target: 80
|
||||
published: 80
|
||||
protocol: tcp
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
- "traefik.http.services.whoami.loadbalancer.server.port=80"
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -1,3 +1,3 @@
|
|||
module github.com/astappiev/traefik-umami-feeder
|
||||
|
||||
go 1.19
|
||||
go 1.21
|
||||
|
|
|
|||
29
readme.md
29
readme.md
|
|
@ -82,15 +82,20 @@ entryPoints:
|
|||
|
||||
## Configuration
|
||||
|
||||
| key | default | type | description |
|
||||
|---------------------|---------|------------|---------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `umamiHost` | - | `string` | Umami server url, reachable from within traefik (container), e.g. `http://umami:3000` |
|
||||
| `umamiToken` | - | `string` | An API Token, used to automatize work with websites, not needed if you provide `websites` |
|
||||
| `umamiUsername` | - | `string` | An alternative to `umamiToken`, you can provide an username and password |
|
||||
| `umamiPassword` | - | `string` | Only in combination with `umamiUsername` |
|
||||
| `umamiTeamId` | - | `string` | In order to organize websites, you can use Umami Teams |
|
||||
| `websites` | - | `map` | A map of hostnames and their associated Umami IDs. Can also be used to override or extend fetched websites |
|
||||
| `createNewWebsites` | false | `bool` | If set to `true`, will try to create a new website on Umami, if domain not found there |
|
||||
| `trackAllResources` | false | `bool` | Defines whether all requests for any resource should be tracked. By default, only requests that are believed to contain content are tracked |
|
||||
| `trackExtensions` | | `string[]` | Defines an alternative list of file extensions that should be tracked |
|
||||
| `debug` | false | `bool` | Something doesn't work? Set to `true` to see more logs (plugins doesn't have access to Traefik's log level) |
|
||||
| key | default | type | description |
|
||||
|---------------------|-------------|------------|-----------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `disabled` | false | `bool` | Set to `true` to disable the plugin |
|
||||
| `debug` | false | `bool` | Something doesn't work? Set to `true` to see more logs (plugins doesn't have access to Traefik's log level) |
|
||||
| `umamiHost` | - | `string` | Umami server url, reachable from within traefik (container), e.g. `http://umami:3000` |
|
||||
| `umamiToken` | - | `string` | An API Token, used to automatize work with websites, not needed if you provide `websites` |
|
||||
| `umamiUsername` | - | `string` | An alternative to `umamiToken`, you can provide an username and password |
|
||||
| `umamiPassword` | - | `string` | Only in combination with `umamiUsername` |
|
||||
| `umamiTeamId` | - | `string` | In order to organize websites, you can use Umami Teams |
|
||||
| `websites` | - | `map` | A map of hostnames and their associated Umami IDs. Can also be used to override or extend fetched websites |
|
||||
| `createNewWebsites` | false | `bool` | If set to `true`, will try to create a new website on Umami, if domain not found there |
|
||||
| `trackAllResources` | false | `bool` | Defines whether all requests for any resource should be tracked. By default, only requests that are believed to contain content are tracked |
|
||||
| `trackExtensions` | | `string[]` | Defines an alternative list of file extensions that should be tracked |
|
||||
| `ignoreUserAgents` | | `string[]` | A list of user agents that should be ignored from tracking, e.g. `["Googlebot", "Uptime-Kuma"]` (matched with `strings.Contains`) |
|
||||
| `ignoreURLs` | | `string[]` | A list of URLs that should be ignored from tracking, e.g. `["/health", "https?://[^/]+/health$"]` (matched with `regexp.Compile.MatchString`) |
|
||||
| `ignoreIPs` | | `string[]` | A list of IPs that should be ignored from tracking, e.g. `["127.0.0.1", "10.0.0.1/16"]` (matched with `netip.ParsePrefix.Contains`) |
|
||||
| `headerIp` | `X-Real-Ip` | `string` | The header to use to get the real IP address of the client, in case it's forwarded by a proxy |
|
||||
|
|
|
|||
129
umami.go
129
umami.go
|
|
@ -8,6 +8,7 @@ import (
|
|||
"net/netip"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
|
@ -42,9 +43,11 @@ type Config struct {
|
|||
// TrackExtensions defines an alternative list of file extensions that should be tracked.
|
||||
TrackExtensions []string `json:"trackExtensions"`
|
||||
|
||||
// IgnoreUserAgents is a list of user agents that should be ignored.
|
||||
// IgnoreUserAgents is a list of user agents to ignore.
|
||||
IgnoreUserAgents []string `json:"ignoreUserAgents"`
|
||||
// IgnoreIPs is a list of IPs or CIDRs that should be ignored.
|
||||
// IgnoreURLs is a list of request urls to ignore, each string is converted to RegExp and urls matched against it.
|
||||
IgnoreURLs []string `json:"ignoreURLs"`
|
||||
// IgnoreIPs is a list of IPs or CIDRs to ignore.
|
||||
IgnoreIPs []string `json:"ignoreIPs"`
|
||||
// headerIp Header associated to real IP
|
||||
HeaderIp string `json:"headerIp"`
|
||||
|
|
@ -69,6 +72,7 @@ func CreateConfig() *Config {
|
|||
TrackExtensions: []string{},
|
||||
|
||||
IgnoreUserAgents: []string{},
|
||||
IgnoreURLs: []string{},
|
||||
IgnoreIPs: []string{},
|
||||
HeaderIp: "X-Real-Ip",
|
||||
}
|
||||
|
|
@ -92,6 +96,7 @@ type UmamiFeeder struct {
|
|||
trackExtensions []string
|
||||
|
||||
ignoreUserAgents []string
|
||||
ignoreRegexps []regexp.Regexp
|
||||
ignorePrefixes []netip.Prefix
|
||||
headerIp string
|
||||
}
|
||||
|
|
@ -116,42 +121,31 @@ func New(ctx context.Context, next http.Handler, config *Config, name string) (h
|
|||
trackExtensions: config.TrackExtensions,
|
||||
|
||||
ignoreUserAgents: config.IgnoreUserAgents,
|
||||
ignoreRegexps: []regexp.Regexp{},
|
||||
ignorePrefixes: []netip.Prefix{},
|
||||
headerIp: config.HeaderIp,
|
||||
}
|
||||
|
||||
if !h.isDisabled {
|
||||
err := h.verifyConfig(config)
|
||||
err := h.connect(config)
|
||||
if err != nil {
|
||||
h.error(err.Error())
|
||||
h.error("due to the error, the Umami plugin is disabled")
|
||||
h.error("unable to connect to Umami, the plugin is disabled")
|
||||
h.isDisabled = true
|
||||
}
|
||||
}
|
||||
|
||||
if len(config.IgnoreIPs) > 0 {
|
||||
for _, ignoreIp := range config.IgnoreIPs {
|
||||
network, err := netip.ParsePrefix(ignoreIp)
|
||||
if err != nil {
|
||||
network, err = netip.ParsePrefix(ignoreIp + "/32")
|
||||
}
|
||||
|
||||
if err != nil || !network.IsValid() {
|
||||
if err != nil {
|
||||
h.error(err.Error())
|
||||
}
|
||||
h.error(fmt.Sprintf("invalid ignoreIp given %s, this param accepts only IP addresses or CIRD in a format 10.0.0.1/16", ignoreIp))
|
||||
h.isDisabled = true
|
||||
} else {
|
||||
h.ignorePrefixes = append(h.ignorePrefixes, network)
|
||||
}
|
||||
err = h.verifyConfig(config)
|
||||
if err != nil {
|
||||
h.error(err.Error())
|
||||
h.error("configuration error, the plugin is disabled")
|
||||
h.isDisabled = true
|
||||
}
|
||||
}
|
||||
|
||||
return h, nil
|
||||
}
|
||||
|
||||
func (h *UmamiFeeder) verifyConfig(config *Config) error {
|
||||
func (h *UmamiFeeder) connect(config *Config) error {
|
||||
if h.umamiHost == "" {
|
||||
return fmt.Errorf("`umamiHost` is not set")
|
||||
}
|
||||
|
|
@ -193,28 +187,45 @@ func (h *UmamiFeeder) verifyConfig(config *Config) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (h *UmamiFeeder) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
|
||||
if !h.isDisabled {
|
||||
if h.shouldTrack(req) {
|
||||
go h.trackRequest(req)
|
||||
} else {
|
||||
h.debug("ignoring request to %s%s", req.Host, req.URL)
|
||||
func (h *UmamiFeeder) verifyConfig(config *Config) error {
|
||||
if len(config.IgnoreIPs) > 0 {
|
||||
for _, ignoreIp := range config.IgnoreIPs {
|
||||
network, err := netip.ParsePrefix(ignoreIp)
|
||||
if err != nil {
|
||||
network, err = netip.ParsePrefix(ignoreIp + "/32")
|
||||
}
|
||||
|
||||
if err != nil || !network.IsValid() {
|
||||
return fmt.Errorf("invalid ignoreIp given %s: %w", ignoreIp, err)
|
||||
}
|
||||
|
||||
h.ignorePrefixes = append(h.ignorePrefixes, network)
|
||||
}
|
||||
}
|
||||
|
||||
if len(config.IgnoreURLs) > 0 {
|
||||
for _, location := range config.IgnoreURLs {
|
||||
r, err := regexp.Compile(location)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to compile ignoreURL %s: %w", location, err)
|
||||
}
|
||||
|
||||
h.ignoreRegexps = append(h.ignoreRegexps, *r)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *UmamiFeeder) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
|
||||
if !h.isDisabled && h.shouldTrack(req) {
|
||||
go h.trackRequest(req)
|
||||
}
|
||||
|
||||
h.next.ServeHTTP(rw, req)
|
||||
}
|
||||
|
||||
func (h *UmamiFeeder) shouldTrack(req *http.Request) bool {
|
||||
if len(h.ignoreUserAgents) > 0 {
|
||||
userAgent := req.UserAgent()
|
||||
for _, disabledUserAgent := range h.ignoreUserAgents {
|
||||
if strings.Contains(userAgent, disabledUserAgent) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(h.ignorePrefixes) > 0 {
|
||||
requestIp := req.Header.Get(h.headerIp)
|
||||
if requestIp == "" {
|
||||
|
|
@ -229,12 +240,34 @@ func (h *UmamiFeeder) shouldTrack(req *http.Request) bool {
|
|||
|
||||
for _, prefix := range h.ignorePrefixes {
|
||||
if prefix.Contains(ip) {
|
||||
h.debug("ignoring IP %s", ip)
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(h.ignoreUserAgents) > 0 {
|
||||
userAgent := req.UserAgent()
|
||||
for _, disabledUserAgent := range h.ignoreUserAgents {
|
||||
if strings.Contains(userAgent, disabledUserAgent) {
|
||||
h.debug("ignoring user-agent %s", userAgent)
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(h.ignoreRegexps) > 0 {
|
||||
requestURL := req.URL.String()
|
||||
for _, r := range h.ignoreRegexps {
|
||||
if r.MatchString(requestURL) {
|
||||
h.debug("ignoring location %s", requestURL)
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !h.shouldTrackResource(req.URL.Path) {
|
||||
h.debug("ignoring resource %s", req.URL.Path)
|
||||
return false
|
||||
}
|
||||
|
||||
|
|
@ -247,6 +280,7 @@ func (h *UmamiFeeder) shouldTrack(req *http.Request) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
h.debug("ignoring domain %s", hostname)
|
||||
return false
|
||||
}
|
||||
|
||||
|
|
@ -269,18 +303,7 @@ func (h *UmamiFeeder) shouldTrackResource(url string) bool {
|
|||
|
||||
// Check if the suffix is regarded to be "content".
|
||||
switch pathExt {
|
||||
case ".htm":
|
||||
case ".html":
|
||||
case ".xhtml":
|
||||
case ".jsf":
|
||||
case ".md":
|
||||
case ".php":
|
||||
case ".rss":
|
||||
case ".rtf":
|
||||
case ".txt":
|
||||
case ".xml":
|
||||
case ".pdf":
|
||||
case "":
|
||||
case "", ".htm", ".html", ".xhtml", ".jsf", ".md", ".php", ".rss", ".rtf", ".txt", ".xml", ".pdf":
|
||||
return true
|
||||
}
|
||||
|
||||
|
|
@ -314,15 +337,15 @@ func (h *UmamiFeeder) trackRequest(req *http.Request) {
|
|||
|
||||
func (h *UmamiFeeder) error(message string) {
|
||||
if h.logHandler != nil {
|
||||
time := time.Now().Format("2006-01-02T15:04:05Z")
|
||||
h.logHandler.Printf("%s ERR middlewareName=%s error=\"%s\"", time, h.name, message)
|
||||
now := time.Now().Format("2006-01-02T15:04:05Z")
|
||||
h.logHandler.Printf("%s ERR middlewareName=%s error=\"%s\"", now, h.name, message)
|
||||
}
|
||||
}
|
||||
|
||||
// Arguments are handled in the manner of [fmt.Printf].
|
||||
func (h *UmamiFeeder) debug(format string, v ...any) {
|
||||
if h.logHandler != nil && h.isDebug {
|
||||
time := time.Now().Format("2006-01-02T15:04:05Z")
|
||||
h.logHandler.Printf("%s DBG middlewareName=%s msg=\"%s\"", time, h.name, fmt.Sprintf(format, v...))
|
||||
now := time.Now().Format("2006-01-02T15:04:05Z")
|
||||
h.logHandler.Printf("%s DBG middlewareName=%s msg=\"%s\"", now, h.name, fmt.Sprintf(format, v...))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
145
umami_test.go
Normal file
145
umami_test.go
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
package traefik_umami_feeder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTraefikUmamiFeeder(t *testing.T) {
|
||||
t.Skip("requires a running Umami instance, see /demo/docker-compose.yml")
|
||||
|
||||
cfg := CreateConfig()
|
||||
cfg.UmamiHost = "http://localhost:3000"
|
||||
cfg.UmamiUsername = "admin"
|
||||
cfg.UmamiPassword = "umami"
|
||||
cfg.UmamiTeamId = "8e39c6ad-e44a-4d3e-be98-015db2d62d40"
|
||||
cfg.CreateNewWebsites = true
|
||||
|
||||
ctx := context.Background()
|
||||
next := http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {})
|
||||
|
||||
handler, err := New(ctx, next, cfg, "umami-feeder")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
recorder := httptest.NewRecorder()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:80", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
handler.ServeHTTP(recorder, req)
|
||||
}
|
||||
|
||||
func TestShouldTrackDefault(t *testing.T) {
|
||||
feeder := UmamiFeeder{}
|
||||
|
||||
assertResource(t, feeder, true, "http://localhost")
|
||||
assertResource(t, feeder, true, "http://localhost/about")
|
||||
assertResource(t, feeder, true, "http://localhost/products.html")
|
||||
assertResource(t, feeder, true, "http://localhost/blog.php")
|
||||
assertResource(t, feeder, true, "http://localhost/feed.rss")
|
||||
assertResource(t, feeder, false, "http://localhost/favicon.ico")
|
||||
assertResource(t, feeder, false, "http://localhost/photo.jpg")
|
||||
assertResource(t, feeder, false, "http://localhost/background.png")
|
||||
}
|
||||
|
||||
func assertResource(t *testing.T, plugin UmamiFeeder, expected bool, url string) {
|
||||
if expected != plugin.shouldTrackResource(url) {
|
||||
t.Fatalf("expected %v for %s", expected, url)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldTrackInvalidIp(t *testing.T) {
|
||||
feeder := UmamiFeeder{}
|
||||
err := feeder.verifyConfig(&Config{
|
||||
IgnoreIPs: []string{"127.0.0.1-127.0.0.10"},
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("should have failed with invalid IP")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldTrackIps(t *testing.T) {
|
||||
feeder := UmamiFeeder{createNewWebsites: true}
|
||||
err := feeder.verifyConfig(&Config{
|
||||
IgnoreIPs: []string{"127.0.0.1", "10.0.0.1/24"},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
assertIgnoreIp(t, feeder, true, "192.168.0.1")
|
||||
assertIgnoreIp(t, feeder, false, "127.0.0.1")
|
||||
assertIgnoreIp(t, feeder, false, "10.0.0.1")
|
||||
assertIgnoreIp(t, feeder, false, "10.0.0.255")
|
||||
assertIgnoreIp(t, feeder, true, "10.0.1.1")
|
||||
assertIgnoreIp(t, feeder, true, "10.10.10.1")
|
||||
assertIgnoreIp(t, feeder, true, "1.1.1.1")
|
||||
assertIgnoreIp(t, feeder, true, "8.8.8.8")
|
||||
}
|
||||
|
||||
func assertIgnoreIp(t *testing.T, plugin UmamiFeeder, expected bool, clientIp string) {
|
||||
req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://localhost", nil)
|
||||
req.Header.Set(plugin.headerIp, clientIp)
|
||||
|
||||
if expected != plugin.shouldTrack(req) {
|
||||
t.Fatalf("expected %v for %s", expected, clientIp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldTrackUrls(t *testing.T) {
|
||||
feeder := UmamiFeeder{createNewWebsites: true}
|
||||
err := feeder.verifyConfig(&Config{
|
||||
IgnoreURLs: []string{"https?://[^/]+/health$", "/about"},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
assertIgnoreUrl(t, feeder, false, "http://localhost/health")
|
||||
assertIgnoreUrl(t, feeder, true, "http://localhost/user/health")
|
||||
assertIgnoreUrl(t, feeder, true, "http://localhost/healthcheck")
|
||||
assertIgnoreUrl(t, feeder, true, "http://localhost/")
|
||||
assertIgnoreUrl(t, feeder, false, "http://localhost/about")
|
||||
assertIgnoreUrl(t, feeder, false, "http://localhost/aboutus")
|
||||
assertIgnoreUrl(t, feeder, false, "http://localhost/category/about")
|
||||
assertIgnoreUrl(t, feeder, true, "http://localhost/hello-world")
|
||||
}
|
||||
|
||||
func assertIgnoreUrl(t *testing.T, plugin UmamiFeeder, expected bool, url string) {
|
||||
req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
|
||||
|
||||
if expected != plugin.shouldTrack(req) {
|
||||
t.Fatalf("expected %v for %s", expected, url)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldTrackUserAgents(t *testing.T) {
|
||||
feeder := UmamiFeeder{createNewWebsites: true, ignoreUserAgents: []string{"Googlebot", "Uptime-Kuma"}}
|
||||
|
||||
assertIgnoreUa(t, feeder, true, "Mozilla/5.0 (Windows; Windows NT 6.0; WOW64) Gecko/20100101 Firefox/60.7")
|
||||
assertIgnoreUa(t, feeder, true, "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 10.0; Win64; x64 Trident/6.0)")
|
||||
assertIgnoreUa(t, feeder, true, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
|
||||
assertIgnoreUa(t, feeder, false, "Uptime-Kuma/1.18.5")
|
||||
assertIgnoreUa(t, feeder, false, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/90.0.4430.212 Safari/537.36 Uptime-Kuma/1.23.1")
|
||||
assertIgnoreUa(t, feeder, true, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
|
||||
assertIgnoreUa(t, feeder, false, "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
|
||||
assertIgnoreUa(t, feeder, false, "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
|
||||
}
|
||||
|
||||
func assertIgnoreUa(t *testing.T, plugin UmamiFeeder, expected bool, ua string) {
|
||||
req, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://localhost/", nil)
|
||||
req.Header.Set("User-Agent", ua)
|
||||
|
||||
if expected != plugin.shouldTrack(req) {
|
||||
t.Fatalf("expected %v for %s", expected, ua)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue