traefik-rybbit-feeder/umami.go

package traefik_umami_feeder

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"net/netip"
	"os"
	"path"
	"regexp"
	"strings"
	"time"
)

// Config the plugin configuration.
type Config struct {
	// Disabled disables the plugin.
	Disabled bool `json:"disabled"`
	// Debug enables debug logging, be prepared for flooding.
	Debug bool `json:"debug"`
	// QueueSize defines the size of queue, i.e. the amount of events that are waiting to be submitted to Umami.
	QueueSize int `json:"queueSize"`

	// UmamiHost is the URL of the Umami instance.
	UmamiHost string `json:"umamiHost"`
	// UmamiToken is an API KEY, which is optional, but either UmamiToken or Websites should be set.
	UmamiToken string `json:"umamiToken"`
	// UmamiUsername could be provided as an alternative to UmamiToken, used to retrieve the token.
	UmamiUsername string `json:"umamiUsername"`
	// UmamiPassword is required if UmamiUsername is set.
	UmamiPassword string `json:"umamiPassword"`
	// UmamiTeamId defines a team, which will be used to retrieve the websites.
	UmamiTeamId string `json:"umamiTeamId"`

	// Websites is a map of domain to websiteId, which is required if UmamiToken is not set.
	// If both UmamiToken and Websites are set, Websites will override/extend domains retrieved from the API.
	Websites map[string]string `json:"websites"`
	// CreateNewWebsites when set to true, the plugin will create new websites using API, UmamiToken is required.
	CreateNewWebsites bool `json:"createNewWebsites"`

	// TrackErrors defines whether errors (status codes >= 400) should be tracked.
	TrackErrors bool `json:"trackErrors"`
	// TrackAllResources defines whether all requests for any resource should be tracked.
	// By default, only requests that are believed to contain content are tracked.
	TrackAllResources bool `json:"trackAllResources"`
	// TrackExtensions defines an alternative list of file extensions that should be tracked.
	TrackExtensions []string `json:"trackExtensions"`

	// IgnoreUserAgents is a list of user agents to ignore.
	IgnoreUserAgents []string `json:"ignoreUserAgents"`
	// IgnoreURLs is a list of request urls to ignore, each string is converted to RegExp and urls matched against it.
	IgnoreURLs []string `json:"ignoreURLs"`
	// IgnoreIPs is a list of IPs or CIDRs to ignore.
	IgnoreIPs []string `json:"ignoreIPs"`
	// headerIp Header associated to real IP
	HeaderIp string `json:"headerIp"`
}

// CreateConfig creates the default plugin configuration.
func CreateConfig() *Config {
	return &Config{
		Disabled:    false,
		Debug:       false,
		QueueSize:   1000,
		TrackErrors: false,

		UmamiHost:     "",
		UmamiToken:    "",
		UmamiUsername: "",
		UmamiPassword: "",
		UmamiTeamId:   "",

		Websites:          map[string]string{},
		CreateNewWebsites: false,

		TrackAllResources: false,
		TrackExtensions:   []string{},

		IgnoreUserAgents: []string{},
		IgnoreURLs:       []string{},
		IgnoreIPs:        []string{},
		HeaderIp:         "X-Real-Ip",
	}
}

// UmamiFeeder a UmamiFeeder plugin.
type UmamiFeeder struct {
	next       http.Handler
	name       string
	isDebug    bool
	isDisabled bool
	logHandler *log.Logger
	queue      chan *UmamiPayload

	umamiHost         string
	umamiToken        string
	umamiTeamId       string
	websites          map[string]string
	createNewWebsites bool

	trackErrors       bool
	trackAllResources bool
	trackExtensions   []string

	ignoreUserAgents []string
	ignoreRegexps    []regexp.Regexp
	ignorePrefixes   []netip.Prefix
	headerIp         string
}

// New created a new Demo plugin.
func New(ctx context.Context, next http.Handler, config *Config, name string) (http.Handler, error) {
	// construct
	h := &UmamiFeeder{
		next:       next,
		name:       name,
		isDebug:    config.Debug,
		isDisabled: config.Disabled,
		logHandler: log.New(os.Stdout, "", 0),
		// Umami API does not support batching https://github.com/umami-software/umami/discussions/1473
		queue: make(chan *UmamiPayload, config.QueueSize),

		umamiHost:         config.UmamiHost,
		umamiToken:        config.UmamiToken,
		umamiTeamId:       config.UmamiTeamId,
		websites:          config.Websites,
		createNewWebsites: config.CreateNewWebsites,

		trackErrors:       config.TrackErrors,
		trackAllResources: config.TrackAllResources,
		trackExtensions:   config.TrackExtensions,

		ignoreUserAgents: config.IgnoreUserAgents,
		ignoreRegexps:    []regexp.Regexp{},
		ignorePrefixes:   []netip.Prefix{},
		headerIp:         config.HeaderIp,
	}

	if !h.isDisabled {
		err := h.connect(ctx, config)
		if err != nil {
			h.error("unable to connect to Umami, the plugin is disabled: " + err.Error())
			h.isDisabled = true
		}

		err = h.verifyConfig(config)
		if err != nil {
			h.error("configuration error, the plugin is disabled: " + err.Error())
			h.isDisabled = true
		}

		go h.startWorker(ctx)
	}

	return h, nil
}

func (h *UmamiFeeder) connect(ctx context.Context, config *Config) error {
	if h.umamiHost == "" {
		return fmt.Errorf("`umamiHost` is not set")
	}

	if config.UmamiUsername != "" && config.UmamiPassword != "" {
		token, err := getToken(ctx, h.umamiHost, config.UmamiUsername, config.UmamiPassword)
		if err != nil {
			return fmt.Errorf("failed to get token: %w", err)
		}
		if token == "" {
			return fmt.Errorf("retrieved token is empty")
		}
		h.debug("token received %s", token)
		h.umamiToken = token
	}
	if h.umamiToken == "" && len(h.websites) == 0 {
		return fmt.Errorf("either `umamiToken` or `websites` should be set")
	}
	if h.umamiToken == "" && h.createNewWebsites {
		return fmt.Errorf("`umamiToken` is required to create new websites")
	}

	if h.umamiToken != "" {
		websites, err := fetchWebsites(ctx, h.umamiHost, h.umamiToken, h.umamiTeamId)
		if err != nil {
			return fmt.Errorf("failed to fetch websites: %w", err)
		}

		for _, website := range *websites {
			if _, ok := h.websites[website.Domain]; ok {
				continue
			}

			h.websites[website.Domain] = website.ID
			h.debug("fetched websiteId for: %s", website.Domain)
		}
	}

	return nil
}

func (h *UmamiFeeder) verifyConfig(config *Config) error {
	if len(config.IgnoreIPs) > 0 {
		for _, ignoreIp := range config.IgnoreIPs {
			network, err := netip.ParsePrefix(ignoreIp)
			if err != nil {
				network, err = netip.ParsePrefix(ignoreIp + "/32")
			}

			if err != nil || !network.IsValid() {
				return fmt.Errorf("invalid ignoreIp given %s: %w", ignoreIp, err)
			}

			h.ignorePrefixes = append(h.ignorePrefixes, network)
		}
	}

	if len(config.IgnoreURLs) > 0 {
		for _, location := range config.IgnoreURLs {
			r, err := regexp.Compile(location)
			if err != nil {
				return fmt.Errorf("failed to compile ignoreURL %s: %w", location, err)
			}

			h.ignoreRegexps = append(h.ignoreRegexps, *r)
		}
	}

	return nil
}

func (h *UmamiFeeder) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	if !h.isDisabled && h.shouldTrack(req) {
		// If the resource should be reported, we wrap the response writer and check the status code before reporting
		wrappedResponseWriter := &ResponseWriter{
			ResponseWriter: rw,
			request:        req,
			feeder:         h,
		}

		// Continue with next handler.
		h.next.ServeHTTP(wrappedResponseWriter, req)
		return
	}

	h.next.ServeHTTP(rw, req)
}

func (h *UmamiFeeder) shouldTrack(req *http.Request) bool {
	if len(h.ignorePrefixes) > 0 {
		requestIp := req.Header.Get(h.headerIp)
		if requestIp == "" {
			requestIp = req.RemoteAddr
		}

		ip, err := netip.ParseAddr(requestIp)
		if err != nil {
			h.debug("invalid IP %s", requestIp)
			return false
		}

		for _, prefix := range h.ignorePrefixes {
			if prefix.Contains(ip) {
				h.debug("ignoring IP %s", ip)
				return false
			}
		}
	}

	if len(h.ignoreUserAgents) > 0 {
		userAgent := req.UserAgent()
		for _, disabledUserAgent := range h.ignoreUserAgents {
			if strings.Contains(userAgent, disabledUserAgent) {
				h.debug("ignoring user-agent %s", userAgent)
				return false
			}
		}
	}

	if len(h.ignoreRegexps) > 0 {
		requestURL := req.URL.String()
		for _, r := range h.ignoreRegexps {
			if r.MatchString(requestURL) {
				h.debug("ignoring location %s", requestURL)
				return false
			}
		}
	}

	if !h.shouldTrackResource(req.URL.Path) {
		h.debug("ignoring resource %s", req.URL.Path)
		return false
	}

	if h.createNewWebsites {
		return true
	}

	hostname := parseDomainFromHost(req.Host)
	if _, ok := h.websites[hostname]; ok {
		return true
	}

	h.debug("ignoring domain %s", hostname)
	return false
}

func (h *UmamiFeeder) shouldTrackResource(url string) bool {
	if h.trackAllResources {
		return true
	}

	pathExt := path.Ext(url)

	// If a custom file extension list is defined, check if the resource matches it. If not, do not report.
	if len(h.trackExtensions) > 0 {
		for _, suffix := range h.trackExtensions {
			if suffix == pathExt {
				return true
			}
		}
		return false
	}

	// Check if the suffix is regarded to be "content".
	switch pathExt {
	case "", ".htm", ".html", ".xhtml", ".jsf", ".md", ".php", ".rss", ".rtf", ".txt", ".xml", ".pdf":
		return true
	}

	return false
}

func (h *UmamiFeeder) shouldTrackStatus(statusCode int) (report bool) {
	if statusCode >= 400 {
		if h.trackErrors {
			return true
		}

		h.debug("not reporting %d error", statusCode)
		return false
	}
	return true
}

func (h *UmamiFeeder) error(message string) {
	if h.logHandler != nil {
		now := time.Now().Format("2006-01-02T15:04:05Z")
		h.logHandler.Printf("%s ERR middlewareName=%s error=\"%s\"", now, h.name, message)
	}
}

// Arguments are handled in the manner of [fmt.Printf].
func (h *UmamiFeeder) debug(format string, v ...any) {
	if h.logHandler != nil && h.isDebug {
		now := time.Now().Format("2006-01-02T15:04:05Z")
		h.logHandler.Printf("%s DBG middlewareName=%s msg=\"%s\"", now, h.name, fmt.Sprintf(format, v...))
	}
}