From 85f16f1d0e33a4a11361305dc82c0c9ffcdb161f Mon Sep 17 00:00:00 2001 From: Jan-Lukas Else Date: Wed, 1 May 2019 11:46:52 +0200 Subject: [PATCH] Add Daily Email reports feature (#4) --- .gitignore | 1 + README.md | 55 ++++++++++++++++++--- config.go | 105 +++++++++++++++++++++++---------------- config_test.go | 131 ++++++------------------------------------------- database.go | 6 +-- go.mod | 3 ++ go.sum | 6 +++ main.go | 26 +++++----- reports.go | 58 ++++++++++++++++++++++ 9 files changed, 210 insertions(+), 181 deletions(-) create mode 100644 reports.go diff --git a/.gitignore b/.gitignore index dddcc6c..8d3e048 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .idea/ data/ +config.json diff --git a/README.md b/README.md index 5e72574..19f78ab 100644 --- a/README.md +++ b/README.md @@ -6,27 +6,41 @@ KISSS is really easy to install via Docker. - docker run -d --name kis3 -e ENVVAR=VARVALUE -e ... -p 8080:8080 -v kis3:/app/data kis3/kis3 - -Replace ENVVAR and VARVALUE with the environment variables from the configuration. + docker run -d --name kis3 -p 8080:8080 -v kis3:/app/data -v ${pwd}/config.json:/app/config.json kis3/kis3 Depending on your setup, replace `-p 8080:8080` with your custom port configuration. KISSS listens to port 8080 by default, but you can also change this via the configuration. To persist the data KISSS collects, you should mount a volume or a folder to `/app/data`. When mounting an folder, give writing permissions to UID 100, because it is a non-root image to make it more secure. +You should also mount a configuration file to `/app/config.json`. + It's also possible to use KISSS without Docker, but for that you need to compile it yourself. In the future there will be executables without dependencies available. ## Configuration -You can configure some settings using environment variables: +You can configure some settings using a `config.json` file in the working directory or provide a custom path to it using the `-c` CLI flag: -`PORT` (`8080`): Set the port to which KISSS should listen +`port` (`8080`): Set the port to which KISSS should listen -`DNT` (`true`): Set whether or not KISSS should respect Do-Not-Track headers some browsers send +`dnt` (`true`): Set whether or not KISSS should respect Do-Not-Track headers some browsers send -`DB_PATH` (`data/kis3.db`): Set the path for the SQLite database (relative to the working directory - in the Docker container it's `/app`). +`dbPath` (`data/kis3.db`): Set the path for the SQLite database (relative to the working directory - in the Docker container it's `/app`). -You can make the statistics private and only accessible with authentication by setting both `STATS_USERNAME` and `STATS_PASSWORD` to a username and password. If only one or none is set, the statistics are accessible without authorization and public to anyone. +You can make the statistics private and only accessible with authentication by setting both `statsUsername` and `statsPassword` to a username and password. If only one or none is set, the statistics are accessible without authorization and public to anyone. + +The configuration file can look like this: + +```json +{ + "port": 8080, + "dnt": true, + "dbPath": "data/kis3.db", + "statsUsername": "myusername", + "statsPassword": "mysecretpassword" +} +``` + +If you specify an environment variable (`PORT`, `DNT`, `DB_PATH`, `STATS_USERNAME`, `STATS_PASSWORD`), that will override the settings from the configuration file. ## Add to website @@ -56,6 +70,31 @@ The following filters are available: `format`: the format to represent the data, default is `plain` for a simple plain text list, `json` for a JSON response or `chart` for a chart generated with ChartJS in the browser +## Daily email reports + +KISSS has a feature that can send you daily email reports. It basically requests the statistics and sends the response via email. You can configure it by adding report configurations to the configuration file: + +```json +{ + // Other configurations... + "reports": [ + { + "name": "Daily stats from KISSS", + "time": "15:00", + "query": "view=pages&orderrow=second&order=desc", + "from": "myemailaddress@mydomain.tld", + "to": "myemailaddress@mydomain.tld", + "smtpHost": "mail.mydomain.tld:587", + "smtpUser": "myemailaddress@mydomain.tld", + "smtpPassword": "mysecretpassword" + }, + { + // Additional reports... + } + ] +} +``` + ## License KISSS is licensed under the MIT license, so you can do basically everything with it, but nevertheless, please contribute your improvements to make KISSS better for everyone. See the LICENSE.txt file. diff --git a/config.go b/config.go index 6870060..3e27bc1 100644 --- a/config.go +++ b/config.go @@ -1,68 +1,87 @@ package main import ( + "encoding/json" + "flag" + "io/ioutil" "os" "strconv" ) type config struct { - port string - dnt bool - dbPath string - statsAuth bool - statsUsername string - statsPassword string + Port string `json:"port"` + Dnt bool `json:"dnt"` + DbPath string `json:"dbPath"` + StatsUsername string `json:"statsUsername"` + StatsPassword string `json:"statsPassword"` + Reports []report `json:"reports"` +} + +type report struct { + Name string `json:"name"` + Time string `json:"time"` + To string `json:"to"` + SmtpUser string `json:"smtpUser"` + SmtpPassword string `json:"smtpPassword"` + SmtpHost string `json:"smtpHost"` + From string `json:"from"` + Query string `json:"query"` } var ( - appConfig = &config{} + appConfig = &config{ + Port: "8080", + Dnt: true, + DbPath: "data/kis3.db", + StatsUsername: "", + StatsPassword: "", + } ) func init() { - appConfig.port = port() - appConfig.dnt = dnt() - appConfig.dbPath = dbPath() - appConfig.statsUsername = statsUsername() - appConfig.statsPassword = statsPassword() - appConfig.statsAuth = statsAuth(appConfig) + parseConfigFile(appConfig) + // Replace values that are set via environment vars (to make it compatible with old method) + overwriteEnvVarValues(appConfig) } -func port() string { - port := os.Getenv("PORT") - if len(port) != 0 { - return port - } else { - return "8080" - } -} - -func dnt() bool { - dnt := os.Getenv("DNT") - dntBool, e := strconv.ParseBool(dnt) +func parseConfigFile(appConfig *config) { + configFile := flag.String("c", "config.json", "Config file") + flag.Parse() + configJson, e := ioutil.ReadFile(*configFile) if e != nil { - dntBool = true + return } - return dntBool -} - -func dbPath() (dbPath string) { - dbPath = os.Getenv("DB_PATH") - if len(dbPath) == 0 { - dbPath = "data/kis3.db" + e = json.Unmarshal([]byte(configJson), appConfig) + if e != nil { + return } return } -func statsUsername() (username string) { - username = os.Getenv("STATS_USERNAME") - return +func overwriteEnvVarValues(appConfig *config) { + port, set := os.LookupEnv("PORT") + if set { + appConfig.Port = port + } + dntString, set := os.LookupEnv("DNT") + dntBool, e := strconv.ParseBool(dntString) + if set && e == nil { + appConfig.Dnt = dntBool + } + dbPath, set := os.LookupEnv("DB_PATH") + if set { + appConfig.DbPath = dbPath + } + username, set := os.LookupEnv("STATS_USERNAME") + if set { + appConfig.StatsUsername = username + } + password, set := os.LookupEnv("STATS_PASSWORD") + if set { + appConfig.StatsPassword = password + } } -func statsPassword() (password string) { - password = os.Getenv("STATS_PASSWORD") - return -} - -func statsAuth(ac *config) bool { - return len(ac.statsUsername) > 0 && len(ac.statsPassword) > 0 +func (ac *config) statsAuth() bool { + return len(ac.StatsUsername) > 0 && len(ac.StatsPassword) > 0 } diff --git a/config_test.go b/config_test.go index b528e42..2d6abde 100644 --- a/config_test.go +++ b/config_test.go @@ -1,131 +1,32 @@ package main import ( - "os" "testing" ) -func Test_port(t *testing.T) { +func Test_config_statsAuth(t *testing.T) { + type fields struct { + StatsUsername string + StatsPassword string + } tests := []struct { name string - envVar string - want string - }{ - {name: "default", envVar: "", want: "8080"}, - {name: "custom", envVar: "1234", want: "1234"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _ = os.Setenv("PORT", tt.envVar) - if got := port(); got != tt.want { - t.Errorf("port() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_dnt(t *testing.T) { - tests := []struct { - name string - envVar string + fields fields want bool }{ - {name: "default", envVar: "", want: true}, - {envVar: "true", want: true}, - {envVar: "t", want: true}, - {envVar: "TRUE", want: true}, - {envVar: "1", want: true}, - {envVar: "false", want: false}, - {envVar: "f", want: false}, - {envVar: "0", want: false}, - {envVar: "abc", want: true}, + {"No username nor password", fields{"", ""}, false}, + {"Only username", fields{"abc", ""}, false}, + {"Only password", fields{"", "abc"}, false}, + {"Username and password", fields{"abc", "abc"}, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _ = os.Setenv("DNT", tt.envVar) - if got := dnt(); got != tt.want { - t.Errorf("dnt() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_dbPath(t *testing.T) { - tests := []struct { - name string - envVar string - wantDbPath string - }{ - {name: "default", envVar: "", wantDbPath: "data/kis3.db"}, - {envVar: "kis3.db", wantDbPath: "kis3.db"}, - {envVar: "data.db", wantDbPath: "data.db"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _ = os.Setenv("DB_PATH", tt.envVar) - if gotDbPath := dbPath(); gotDbPath != tt.wantDbPath { - t.Errorf("dbPath() = %v, want %v", gotDbPath, tt.wantDbPath) - } - }) - } -} - -func Test_statsUsername(t *testing.T) { - tests := []struct { - name string - envVar string - wantUsername string - }{ - {name: "default", envVar: "", wantUsername: ""}, - {envVar: "abc", wantUsername: "abc"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _ = os.Setenv("STATS_USERNAME", tt.envVar) - if gotUsername := statsUsername(); gotUsername != tt.wantUsername { - t.Errorf("statsUsername() = %v, want %v", gotUsername, tt.wantUsername) - } - }) - } -} - -func Test_statsPassword(t *testing.T) { - tests := []struct { - name string - envVar string - wantPassword string - }{ - {name: "default", envVar: "", wantPassword: ""}, - {envVar: "def", wantPassword: "def"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _ = os.Setenv("STATS_PASSWORD", tt.envVar) - if gotPassword := statsPassword(); gotPassword != tt.wantPassword { - t.Errorf("statsPassword() = %v, want %v", gotPassword, tt.wantPassword) - } - }) - } -} - -func Test_statsAuth(t *testing.T) { - type args struct { - ac *config - } - tests := []struct { - name string - args args - want bool - }{ - {name: "default", args: struct{ ac *config }{ac: &config{}}, want: false}, - {name: "only username set", args: struct{ ac *config }{ac: &config{statsUsername: "abc"}}, want: false}, - {name: "only password set", args: struct{ ac *config }{ac: &config{statsPassword: "def"}}, want: false}, - {name: "username and password set", args: struct{ ac *config }{ac: &config{statsUsername: "abc", statsPassword: "def"}}, want: true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := statsAuth(tt.args.ac); got != tt.want { - t.Errorf("statsAuth() = %v, want %v", got, tt.want) + ac := &config{ + StatsUsername: tt.fields.StatsUsername, + StatsPassword: tt.fields.StatsPassword, + } + if got := ac.statsAuth(); got != tt.want { + t.Errorf("config.statsAuth() = %v, want %v", got, tt.want) } }) } diff --git a/database.go b/database.go index a25df32..02c091e 100644 --- a/database.go +++ b/database.go @@ -19,10 +19,10 @@ type Database struct { func initDatabase() (database *Database, e error) { database = &Database{} - if _, err := os.Stat(appConfig.dbPath); os.IsNotExist(err) { - _ = os.MkdirAll(filepath.Dir(appConfig.dbPath), os.ModePerm) + if _, err := os.Stat(appConfig.DbPath); os.IsNotExist(err) { + _ = os.MkdirAll(filepath.Dir(appConfig.DbPath), os.ModePerm) } - database.sqlDB, e = sql.Open("sqlite3", appConfig.dbPath) + database.sqlDB, e = sql.Open("sqlite3", appConfig.DbPath) if e != nil { return } diff --git a/go.mod b/go.mod index bdd2d32..4e4a8a4 100644 --- a/go.mod +++ b/go.mod @@ -6,12 +6,15 @@ require ( github.com/go-sql-driver/mysql v1.4.1 // indirect github.com/gobuffalo/packr v1.25.0 // indirect github.com/gobuffalo/packr/v2 v2.2.0 + github.com/google/uuid v1.1.1 // indirect github.com/gorilla/handlers v1.4.0 github.com/gorilla/mux v1.7.1 + github.com/jordan-wright/email v0.0.0-20190218024454-3ea4d25e7cf8 github.com/lib/pq v1.1.0 // indirect github.com/mattn/go-sqlite3 v0.0.0-20190424093727-5994cc52dfa8 github.com/mssola/user_agent v0.5.0 github.com/rubenv/sql-migrate v0.0.0-20190327083759-54bad0a9b051 + github.com/whiteshtef/clockwork v0.0.0-20190417075149-ecf7d9abe8ec github.com/ziutek/mymysql v1.5.4 // indirect google.golang.org/appengine v1.5.0 // indirect gopkg.in/gorp.v1 v1.7.2 // indirect diff --git a/go.sum b/go.sum index 5441111..d3e23f6 100644 --- a/go.sum +++ b/go.sum @@ -24,10 +24,14 @@ github.com/gobuffalo/packr/v2 v2.2.0 h1:Ir9W9XIm9j7bhhkKE9cokvtTl1vBm62A/fene/ZC github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/handlers v1.4.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jordan-wright/email v0.0.0-20190218024454-3ea4d25e7cf8 h1:XMe1IsRiRx3E3M50BhP7327VYF4A9RpCFfhHUFW+IeE= +github.com/jordan-wright/email v0.0.0-20190218024454-3ea4d25e7cf8/go.mod h1:1c7szIrayyPPB/987hsnvNzLushdWf4o/79s3P08L8A= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -55,6 +59,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/whiteshtef/clockwork v0.0.0-20190417075149-ecf7d9abe8ec h1:4mCJZnO75zjolpdsj/ToKe7X1oLWm+JJHwS1ez8BkXY= +github.com/whiteshtef/clockwork v0.0.0-20190417075149-ecf7d9abe8ec/go.mod h1:6o8H8sci2q3QxZ4p/U88ggqZuhY3mg34+WE5BuazLsU= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/main.go b/main.go index 88088d7..6cc34fb 100644 --- a/main.go +++ b/main.go @@ -21,7 +21,9 @@ type kis3 struct { } var ( - app = &kis3{} + app = &kis3{ + staticBox: packr.New("staticFiles", "./static"), + } ) func init() { @@ -30,7 +32,7 @@ func init() { log.Fatal("Database setup failed:", e) } setupRouter() - app.staticBox = packr.New("staticFiles", "./static") + setupReports() } func main() { @@ -49,29 +51,29 @@ func setupRouter() { viewRouter := app.router.PathPrefix("/view").Subrouter() viewRouter.Use(corsHandler) - viewRouter.Path("").HandlerFunc(trackView) + viewRouter.Path("").HandlerFunc(TrackingHandler) - app.router.HandleFunc("/stats", requestStats) + app.router.HandleFunc("/stats", StatsHandler) staticRouter := app.router.PathPrefix("").Subrouter() staticRouter.Use(corsHandler) - staticRouter.HandleFunc("/kis3.js", serveTrackingScript) + staticRouter.HandleFunc("/kis3.js", TrackingScriptHandler) staticRouter.PathPrefix("").Handler(http.HandlerFunc(HelloResponseHandler)) } func startListening() { - port := appConfig.port + port := appConfig.Port addr := ":" + port fmt.Printf("Listening to %s\n", addr) log.Fatal(http.ListenAndServe(addr, app.router)) } -func trackView(w http.ResponseWriter, r *http.Request) { +func TrackingHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate, max-age=0") url := r.URL.Query().Get("url") ref := r.URL.Query().Get("ref") ua := r.Header.Get("User-Agent") - if !(r.Header.Get("DNT") == "1" && appConfig.dnt) { + if !(r.Header.Get("DNT") == "1" && appConfig.Dnt) { go app.db.trackView(url, ref, ua) // run with goroutine for awesome speed! _, _ = fmt.Fprint(w, "true") } @@ -81,7 +83,7 @@ func HelloResponseHandler(w http.ResponseWriter, _ *http.Request) { _, _ = fmt.Fprint(w, "Hello from KISSS") } -func serveTrackingScript(w http.ResponseWriter, r *http.Request) { +func TrackingScriptHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/javascript") w.Header().Set("Cache-Control", "public, max-age=432000") // 5 days filename := "kis3.js" @@ -97,10 +99,10 @@ func serveTrackingScript(w http.ResponseWriter, r *http.Request) { http.ServeContent(w, r, filename, stat.ModTime(), file) } -func requestStats(w http.ResponseWriter, r *http.Request) { +func StatsHandler(w http.ResponseWriter, r *http.Request) { // Require authentication - if appConfig.statsAuth { - if !helpers.CheckAuth(w, r, appConfig.statsUsername, appConfig.statsPassword) { + if appConfig.statsAuth() { + if !helpers.CheckAuth(w, r, appConfig.StatsUsername, appConfig.StatsPassword) { return } } diff --git a/reports.go b/reports.go new file mode 100644 index 0000000..7eebaa5 --- /dev/null +++ b/reports.go @@ -0,0 +1,58 @@ +package main + +import ( + "fmt" + "github.com/jordan-wright/email" + "github.com/whiteshtef/clockwork" + "io/ioutil" + "net" + "net/http" + "net/smtp" +) + +func setupReports() { + scheduler := clockwork.NewScheduler() + for _, r := range appConfig.Reports { + scheduler.Schedule().Every().Day().At(r.Time).Do(func() { + executeReport(&r) + }) + } + go scheduler.Run() +} + +func executeReport(r *report) { + fmt.Println("Execute report:", r.Name) + req, e := http.NewRequest("GET", "http://localhost:"+appConfig.Port+"/stats?"+r.Query, nil) + if e != nil { + fmt.Println("Executing report failed:", e) + return + } + req.SetBasicAuth(appConfig.StatsUsername, appConfig.StatsPassword) + res, e := http.DefaultClient.Do(req) + if e != nil { + fmt.Println("Executing report failed:", e) + return + } + body, e := ioutil.ReadAll(res.Body) + if e != nil { + fmt.Println("Executing report failed:", e) + return + } + sendMail(r, body) +} + +func sendMail(r *report, content []byte) { + smtpHostNoPort, _, _ := net.SplitHostPort(r.SmtpHost) + mail := email.NewEmail() + mail.From = r.From + mail.To = []string{r.To} + mail.Subject = "KISSS report: " + r.Name + mail.Text = content + e := mail.Send(r.SmtpHost, smtp.PlainAuth("", r.SmtpUser, r.SmtpPassword, smtpHostNoPort)) + if e != nil { + fmt.Println("Sending report failed:", e) + return + } else { + fmt.Println("Report sent") + } +}