diff --git a/.vscode/tasks.json b/.vscode/tasks.json index c3e88d5..a2a0a4b 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -4,7 +4,12 @@ { "label": "Build", "type": "shell", - "command": "go build --tags \"libsqlite3 linux sqlite_fts5\"" + "command": "go build --tags \"libsqlite3 linux sqlite_fts5\"", + "problemMatcher": [], + "group": { + "kind": "build", + "isDefault": true + } } ] } \ No newline at end of file diff --git a/check.go b/check.go new file mode 100644 index 0000000..9dfa3e4 --- /dev/null +++ b/check.go @@ -0,0 +1,100 @@ +package main + +import ( + "crypto/tls" + "fmt" + "io" + "log" + "net/http" + "strings" + "sync" + "time" +) + +func checkAllExternalLinks() { + allPosts, err := getPosts(&postsRequestConfig{status: statusPublished}) + if err != nil { + log.Println(err.Error()) + return + } + wg := new(sync.WaitGroup) + linkChan := make(chan stringPair) + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: &http.Transport{ + DisableKeepAlives: true, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + } + responses := map[string]int{} + rm := sync.RWMutex{} + for i := 0; i < 20; i++ { + go func() { + defer wg.Done() + wg.Add(1) + for postLinkPair := range linkChan { + rm.RLock() + _, ok := responses[postLinkPair.second] + rm.RUnlock() + if !ok { + req, err := http.NewRequest(http.MethodGet, postLinkPair.second, nil) + if err != nil { + fmt.Println(err.Error()) + continue + } + // User-Agent from Tor + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.5") + resp, err := client.Do(req) + if err != nil { + fmt.Println(postLinkPair.second+" ("+postLinkPair.first+"):", err.Error()) + continue + } + status := resp.StatusCode + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + rm.Lock() + responses[postLinkPair.second] = status + rm.Unlock() + } + rm.RLock() + if response, ok := responses[postLinkPair.second]; ok && !checkSuccessStatus(response) { + fmt.Println(postLinkPair.second+" ("+postLinkPair.first+"):", response) + } + rm.RUnlock() + } + }() + } + err = getExternalLinks(allPosts, linkChan) + if err != nil { + log.Println(err.Error()) + return + } + wg.Wait() +} + +func checkSuccessStatus(status int) bool { + return status >= 200 && status < 400 +} + +func getExternalLinks(posts []*post, linkChan chan<- stringPair) error { + wg := new(sync.WaitGroup) + for _, p := range posts { + wg.Add(1) + go func(p *post) { + defer wg.Done() + links, _ := allLinksFromHTML(strings.NewReader(string(p.absoluteHTML())), p.fullURL()) + for _, link := range links { + if !strings.HasPrefix(link, appConfig.Server.PublicAddress) { + linkChan <- stringPair{p.fullURL(), link} + } + } + }(p) + } + wg.Wait() + close(linkChan) + return nil +} diff --git a/go.mod b/go.mod index c913fea..4404b10 100644 --- a/go.mod +++ b/go.mod @@ -58,9 +58,9 @@ require ( golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect golang.org/x/mod v0.4.1 // indirect - golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c // indirect + golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 // indirect + golang.org/x/sys v0.0.0-20210402192133-700132347e07 // indirect golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect golang.org/x/text v0.3.6 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/go.sum b/go.sum index eea526d..688661b 100644 --- a/go.sum +++ b/go.sum @@ -454,8 +454,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 h1:rF3Ohx8DRyl8h2zw9qojyLHLhrJpEMgyPOImREEryf0= -golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210402192133-700132347e07 h1:4k6HsQjxj6hVMsI2Vf0yKlzt5lXxZsMW1q0zaq2k8zY= +golang.org/x/sys v0.0.0-20210402192133-700132347e07/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/http.go b/http.go index 8f07105..f17d606 100644 --- a/http.go +++ b/http.go @@ -17,6 +17,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" servertiming "github.com/mitchellh/go-server-timing" + "golang.org/x/net/context" ) const ( @@ -59,20 +60,19 @@ func startServer() (err error) { finalHandler = logMiddleware(finalHandler) } // Create routers that don't change - err = buildStaticHandlersRouters() - if err != nil { - return + if err = buildStaticHandlersRouters(); err != nil { + return err } // Load router - err = reloadRouter() - if err != nil { - return + if err = reloadRouter(); err != nil { + return err } // Start Onion service if appConfig.Server.Tor { go func() { - torErr := startOnionService(finalHandler) - log.Println("Tor failed:", torErr.Error()) + if err := startOnionService(finalHandler); err != nil { + log.Println("Tor failed:", err.Error()) + } }() } // Start server @@ -81,20 +81,30 @@ func startServer() (err error) { ReadTimeout: 5 * time.Minute, WriteTimeout: 5 * time.Minute, } + go onShutdown(func() { + toc, c := context.WithTimeout(context.Background(), 5*time.Second) + _ = s.Shutdown(toc) + c() + }) if appConfig.Server.PublicHTTPS { // Configure certmagic.Default.Storage = &certmagic.FileStorage{Path: "data/https"} certmagic.DefaultACME.Email = appConfig.Server.LetsEncryptMail certmagic.DefaultACME.CA = certmagic.LetsEncryptProductionCA - // Start HTTP server for TLS verification and redirect + // Start HTTP server for redirects httpServer := &http.Server{ Addr: ":http", Handler: http.HandlerFunc(redirectToHttps), ReadTimeout: 5 * time.Minute, WriteTimeout: 5 * time.Minute, } + go onShutdown(func() { + toc, c := context.WithTimeout(context.Background(), 5*time.Second) + _ = httpServer.Shutdown(toc) + c() + }) go func() { - if err := httpServer.ListenAndServe(); err != nil { + if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { log.Println("Failed to start HTTP server:", err.Error()) } }() @@ -108,12 +118,16 @@ func startServer() (err error) { if e != nil { return e } - err = s.Serve(listener) + if err = s.Serve(listener); err != nil && err != http.ErrServerClosed { + return err + } } else { s.Addr = ":" + strconv.Itoa(appConfig.Server.Port) - err = s.ListenAndServe() + if err = s.ListenAndServe(); err != nil && err != http.ErrServerClosed { + return err + } } - return + return nil } func redirectToHttps(w http.ResponseWriter, r *http.Request) { diff --git a/main.go b/main.go index cd87b61..b1810f1 100644 --- a/main.go +++ b/main.go @@ -4,10 +4,8 @@ import ( "flag" "log" "os" - "os/signal" "runtime" "runtime/pprof" - "syscall" "github.com/pquerna/otp/totp" ) @@ -16,6 +14,8 @@ var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to `file`") var memprofile = flag.String("memprofile", "", "write memory profile to `file`") func main() { + var err error + // Init CPU profiling flag.Parse() if *cpuprofile != "" { @@ -32,25 +32,22 @@ func main() { // Initialize config log.Println("Initialize configuration...") - err := initConfig() - if err != nil { - log.Fatal(err) + if err = initConfig(); err != nil { + log.Fatalln("Failed to init config:", err.Error()) } - // Small tools - if len(os.Args) >= 2 { - if os.Args[1] == "totp-secret" { - key, err := totp.Generate(totp.GenerateOpts{ - Issuer: appConfig.Server.PublicAddress, - AccountName: appConfig.User.Nick, - }) - if err != nil { - log.Fatal(err.Error()) - return - } - log.Println("TOTP-Secret:", key.Secret()) + // Small tools before init + if len(os.Args) >= 2 && os.Args[1] == "totp-secret" { + key, err := totp.Generate(totp.GenerateOpts{ + Issuer: appConfig.Server.PublicAddress, + AccountName: appConfig.User.Nick, + }) + if err != nil { + log.Fatalln(err.Error()) return } + log.Println("TOTP-Secret:", key.Secret()) + return } // Init regular garbage collection @@ -58,54 +55,54 @@ func main() { // Execute pre-start hooks preStartHooks() - // Initialize everything else + + // Initialize database and markdown log.Println("Initialize database...") - err = initDatabase() - if err != nil { - log.Fatal(err) + if err = initDatabase(); err != nil { + log.Fatalln("Failed to init database:", err.Error()) return } log.Println("Initialize server components...") - initMinify() initMarkdown() - err = initTemplateAssets() // Needs minify - if err != nil { - log.Fatal(err) + + // Link check tool after init of markdown + if len(os.Args) >= 2 && os.Args[1] == "check" { + checkAllExternalLinks() return } - err = initTemplateStrings() - if err != nil { - log.Fatal(err) + + // More initializations + initMinify() + if err = initTemplateAssets(); err != nil { // Needs minify + log.Fatalln("Failed to init template assets:", err.Error()) return } - err = initRendering() // Needs assets - if err != nil { - log.Fatal(err) + if err = initTemplateStrings(); err != nil { + log.Fatalln("Failed to init template translations:", err.Error()) return } - err = initCache() - if err != nil { - log.Fatal(err) + if err = initRendering(); err != nil { // Needs assets and minify + log.Fatalln("Failed to init HTML rendering:", err.Error()) return } - err = initRegexRedirects() - if err != nil { - log.Fatal(err) + if err = initCache(); err != nil { + log.Fatalln("Failed to init HTTP cache:", err.Error()) return } - err = initHTTPLog() - if err != nil { - log.Fatal(err) + if err = initRegexRedirects(); err != nil { + log.Fatalln("Failed to init redirects:", err.Error()) return } - err = initActivityPub() - if err != nil { - log.Fatal(err) + if err = initHTTPLog(); err != nil { + log.Fatal("Failed to init HTTP logging:", err.Error()) return } - err = initWebmention() - if err != nil { - log.Fatal(err) + if err = initActivityPub(); err != nil { + log.Fatalln("Failed to init ActivityPub:", err.Error()) + return + } + if err = initWebmention(); err != nil { + log.Fatalln("Failed to init webmention support:", err.Error()) return } initTelegram() @@ -113,42 +110,37 @@ func main() { // Start cron hooks startHourlyHooks() - // Prepare graceful shutdown - quit := make(chan os.Signal, 1) - // Start the server - go func() { - log.Println("Starting server...") - err = startServer() - if err != nil { - log.Println("Failed to start server:") - log.Println(err) - } - quit <- os.Interrupt - }() + log.Println("Starting server...") + err = startServer() + if err != nil { + log.Fatalln("Failed to start server:", err.Error()) + return + } + log.Println("Stopped server(s)") - // Graceful shutdown - signal.Notify(quit, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) - <-quit - log.Println("Stopping...") + // Wait till everything is shutdown + waitForShutdown() + + // Close DB + if err = closeDb(); err != nil { + log.Fatalln("Failed to close DB:", err.Error()) + return + } + log.Println("Closed Database") // Write memory profile if *memprofile != "" { f, err := os.Create(*memprofile) if err != nil { - log.Fatal("could not create memory profile: ", err) + log.Fatalln("could not create memory profile: ", err.Error()) + return } defer f.Close() runtime.GC() if err := pprof.WriteHeapProfile(f); err != nil { - log.Fatal("could not write memory profile: ", err) + log.Fatalln("could not write memory profile: ", err.Error()) + return } } - - // Close DB - err = closeDb() - if err != nil { - log.Fatal(err) - return - } } diff --git a/shutdown.go b/shutdown.go new file mode 100644 index 0000000..a5ed9ff --- /dev/null +++ b/shutdown.go @@ -0,0 +1,23 @@ +package main + +import ( + "os" + "os/signal" + "sync" + "syscall" +) + +var shutdownWg sync.WaitGroup + +func onShutdown(f func()) { + defer shutdownWg.Done() + shutdownWg.Add(1) + quit := make(chan os.Signal, 1) + signal.Notify(quit, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) + <-quit + f() +} + +func waitForShutdown() { + shutdownWg.Wait() +} diff --git a/tor.go b/tor.go index 07d5e0a..954767d 100644 --- a/tor.go +++ b/tor.go @@ -82,5 +82,13 @@ func startOnionService(h http.Handler) error { ReadTimeout: 5 * time.Minute, WriteTimeout: 5 * time.Minute, } - return s.Serve(onion) + go onShutdown(func() { + toc, c := context.WithTimeout(context.Background(), 5*time.Second) + _ = s.Shutdown(toc) + c() + }) + if err = s.Serve(onion); err != nil && err != http.ErrServerClosed { + return err + } + return nil } diff --git a/utils.go b/utils.go index 6652328..20b58d7 100644 --- a/utils.go +++ b/utils.go @@ -166,3 +166,7 @@ func dateFormat(date string, format string) string { } return d.Local().Format(format) } + +type stringPair struct { + first, second string +}