Command "check" to check for broken external links and improve graceful shutdown

.vscode/tasks.json

@@ -4,7 +4,12 @@
       {
         "label": "Build",
         "type": "shell",
-        "command": "go build --tags \"libsqlite3 linux sqlite_fts5\""
+        "command": "go build --tags \"libsqlite3 linux sqlite_fts5\"",
+        "problemMatcher": [],
+        "group": {
+          "kind": "build",
+          "isDefault": true
+        }
       }
     ]
   }

check.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+)
+
+func checkAllExternalLinks() {
+	allPosts, err := getPosts(&postsRequestConfig{status: statusPublished})
+	if err != nil {
+		log.Println(err.Error())
+		return
+	}
+	wg := new(sync.WaitGroup)
+	linkChan := make(chan stringPair)
+	client := &http.Client{
+		Timeout: 30 * time.Second,
+		Transport: &http.Transport{
+			DisableKeepAlives: true,
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		},
+	}
+	responses := map[string]int{}
+	rm := sync.RWMutex{}
+	for i := 0; i < 20; i++ {
+		go func() {
+			defer wg.Done()
+			wg.Add(1)
+			for postLinkPair := range linkChan {
+				rm.RLock()
+				_, ok := responses[postLinkPair.second]
+				rm.RUnlock()
+				if !ok {
+					req, err := http.NewRequest(http.MethodGet, postLinkPair.second, nil)
+					if err != nil {
+						fmt.Println(err.Error())
+						continue
+					}
+					// User-Agent from Tor
+					req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0")
+					req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+					req.Header.Set("Accept-Language", "en-US,en;q=0.5")
+					resp, err := client.Do(req)
+					if err != nil {
+						fmt.Println(postLinkPair.second+" ("+postLinkPair.first+"):", err.Error())
+						continue
+					}
+					status := resp.StatusCode
+					_, _ = io.Copy(io.Discard, resp.Body)
+					resp.Body.Close()
+					rm.Lock()
+					responses[postLinkPair.second] = status
+					rm.Unlock()
+				}
+				rm.RLock()
+				if response, ok := responses[postLinkPair.second]; ok && !checkSuccessStatus(response) {
+					fmt.Println(postLinkPair.second+" ("+postLinkPair.first+"):", response)
+				}
+				rm.RUnlock()
+			}
+		}()
+	}
+	err = getExternalLinks(allPosts, linkChan)
+	if err != nil {
+		log.Println(err.Error())
+		return
+	}
+	wg.Wait()
+}
+
+func checkSuccessStatus(status int) bool {
+	return status >= 200 && status < 400
+}
+
+func getExternalLinks(posts []*post, linkChan chan<- stringPair) error {
+	wg := new(sync.WaitGroup)
+	for _, p := range posts {
+		wg.Add(1)
+		go func(p *post) {
+			defer wg.Done()
+			links, _ := allLinksFromHTML(strings.NewReader(string(p.absoluteHTML())), p.fullURL())
+			for _, link := range links {
+				if !strings.HasPrefix(link, appConfig.Server.PublicAddress) {
+					linkChan <- stringPair{p.fullURL(), link}
+				}
+			}
+		}(p)
+	}
+	wg.Wait()
+	close(linkChan)
+	return nil
+}

go.mod

@@ -58,9 +58,9 @@ require (
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect
 	golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
 	golang.org/x/mod v0.4.1 // indirect
-	golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c // indirect
+	golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 // indirect
+	golang.org/x/sys v0.0.0-20210402192133-700132347e07 // indirect
 	golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
 	golang.org/x/text v0.3.6 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect

go.sum

@@ -454,8 +454,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 h1:rF3Ohx8DRyl8h2zw9qojyLHLhrJpEMgyPOImREEryf0=
-golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210402192133-700132347e07 h1:4k6HsQjxj6hVMsI2Vf0yKlzt5lXxZsMW1q0zaq2k8zY=
+golang.org/x/sys v0.0.0-20210402192133-700132347e07/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=
 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

http.go

@@ -17,6 +17,7 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 	servertiming "github.com/mitchellh/go-server-timing"
+	"golang.org/x/net/context"
 )
 
 const (
@@ -59,20 +60,19 @@ func startServer() (err error) {
 		finalHandler = logMiddleware(finalHandler)
 	}
 	// Create routers that don't change
-	err = buildStaticHandlersRouters()
-	if err != nil {
-		return
+	if err = buildStaticHandlersRouters(); err != nil {
+		return err
 	}
 	// Load router
-	err = reloadRouter()
-	if err != nil {
-		return
+	if err = reloadRouter(); err != nil {
+		return err
 	}
 	// Start Onion service
 	if appConfig.Server.Tor {
 		go func() {
-			torErr := startOnionService(finalHandler)
-			log.Println("Tor failed:", torErr.Error())
+			if err := startOnionService(finalHandler); err != nil {
+				log.Println("Tor failed:", err.Error())
+			}
 		}()
 	}
 	// Start server
@@ -81,20 +81,30 @@ func startServer() (err error) {
 		ReadTimeout:  5 * time.Minute,
 		WriteTimeout: 5 * time.Minute,
 	}
+	go onShutdown(func() {
+		toc, c := context.WithTimeout(context.Background(), 5*time.Second)
+		_ = s.Shutdown(toc)
+		c()
+	})
 	if appConfig.Server.PublicHTTPS {
 		// Configure
 		certmagic.Default.Storage = &certmagic.FileStorage{Path: "data/https"}
 		certmagic.DefaultACME.Email = appConfig.Server.LetsEncryptMail
 		certmagic.DefaultACME.CA = certmagic.LetsEncryptProductionCA
-		// Start HTTP server for TLS verification and redirect
+		// Start HTTP server for redirects
 		httpServer := &http.Server{
 			Addr:         ":http",
 			Handler:      http.HandlerFunc(redirectToHttps),
 			ReadTimeout:  5 * time.Minute,
 			WriteTimeout: 5 * time.Minute,
 		}
+		go onShutdown(func() {
+			toc, c := context.WithTimeout(context.Background(), 5*time.Second)
+			_ = httpServer.Shutdown(toc)
+			c()
+		})
 		go func() {
-			if err := httpServer.ListenAndServe(); err != nil {
+			if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
 				log.Println("Failed to start HTTP server:", err.Error())
 			}
 		}()
@@ -108,12 +118,16 @@ func startServer() (err error) {
 		if e != nil {
 			return e
 		}
-		err = s.Serve(listener)
+		if err = s.Serve(listener); err != nil && err != http.ErrServerClosed {
+			return err
+		}
 	} else {
 		s.Addr = ":" + strconv.Itoa(appConfig.Server.Port)
-		err = s.ListenAndServe()
+		if err = s.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			return err
+		}
 	}
-	return
+	return nil
 }
 
 func redirectToHttps(w http.ResponseWriter, r *http.Request) {

main.go

@@ -4,10 +4,8 @@ import (
 	"flag"
 	"log"
 	"os"
-	"os/signal"
 	"runtime"
 	"runtime/pprof"
-	"syscall"
 
 	"github.com/pquerna/otp/totp"
 )
@@ -16,6 +14,8 @@ var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to `file`")
 var memprofile = flag.String("memprofile", "", "write memory profile to `file`")
 
 func main() {
+	var err error
+
 	// Init CPU profiling
 	flag.Parse()
 	if *cpuprofile != "" {
@@ -32,25 +32,22 @@ func main() {
 
 	// Initialize config
 	log.Println("Initialize configuration...")
-	err := initConfig()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	// Small tools
-	if len(os.Args) >= 2 {
-		if os.Args[1] == "totp-secret" {
-			key, err := totp.Generate(totp.GenerateOpts{
-				Issuer:      appConfig.Server.PublicAddress,
-				AccountName: appConfig.User.Nick,
-			})
-			if err != nil {
-				log.Fatal(err.Error())
-				return
-			}
-			log.Println("TOTP-Secret:", key.Secret())
+	if err = initConfig(); err != nil {
+		log.Fatalln("Failed to init config:", err.Error())
+	}
+
+	// Small tools before init
+	if len(os.Args) >= 2 && os.Args[1] == "totp-secret" {
+		key, err := totp.Generate(totp.GenerateOpts{
+			Issuer:      appConfig.Server.PublicAddress,
+			AccountName: appConfig.User.Nick,
+		})
+		if err != nil {
+			log.Fatalln(err.Error())
 			return
 		}
+		log.Println("TOTP-Secret:", key.Secret())
+		return
 	}
 
 	// Init regular garbage collection
@@ -58,54 +55,54 @@ func main() {
 
 	// Execute pre-start hooks
 	preStartHooks()
-	// Initialize everything else
+
+	// Initialize database and markdown
 	log.Println("Initialize database...")
-	err = initDatabase()
-	if err != nil {
-		log.Fatal(err)
+	if err = initDatabase(); err != nil {
+		log.Fatalln("Failed to init database:", err.Error())
 		return
 	}
 	log.Println("Initialize server components...")
-	initMinify()
 	initMarkdown()
-	err = initTemplateAssets() // Needs minify
-	if err != nil {
-		log.Fatal(err)
+
+	// Link check tool after init of markdown
+	if len(os.Args) >= 2 && os.Args[1] == "check" {
+		checkAllExternalLinks()
 		return
 	}
-	err = initTemplateStrings()
-	if err != nil {
-		log.Fatal(err)
+
+	// More initializations
+	initMinify()
+	if err = initTemplateAssets(); err != nil { // Needs minify
+		log.Fatalln("Failed to init template assets:", err.Error())
 		return
 	}
-	err = initRendering() // Needs assets
-	if err != nil {
-		log.Fatal(err)
+	if err = initTemplateStrings(); err != nil {
+		log.Fatalln("Failed to init template translations:", err.Error())
 		return
 	}
-	err = initCache()
-	if err != nil {
-		log.Fatal(err)
+	if err = initRendering(); err != nil { // Needs assets and minify
+		log.Fatalln("Failed to init HTML rendering:", err.Error())
 		return
 	}
-	err = initRegexRedirects()
-	if err != nil {
-		log.Fatal(err)
+	if err = initCache(); err != nil {
+		log.Fatalln("Failed to init HTTP cache:", err.Error())
 		return
 	}
-	err = initHTTPLog()
-	if err != nil {
-		log.Fatal(err)
+	if err = initRegexRedirects(); err != nil {
+		log.Fatalln("Failed to init redirects:", err.Error())
 		return
 	}
-	err = initActivityPub()
-	if err != nil {
-		log.Fatal(err)
+	if err = initHTTPLog(); err != nil {
+		log.Fatal("Failed to init HTTP logging:", err.Error())
 		return
 	}
-	err = initWebmention()
-	if err != nil {
-		log.Fatal(err)
+	if err = initActivityPub(); err != nil {
+		log.Fatalln("Failed to init ActivityPub:", err.Error())
+		return
+	}
+	if err = initWebmention(); err != nil {
+		log.Fatalln("Failed to init webmention support:", err.Error())
 		return
 	}
 	initTelegram()
@@ -113,42 +110,37 @@ func main() {
 	// Start cron hooks
 	startHourlyHooks()
 
-	// Prepare graceful shutdown
-	quit := make(chan os.Signal, 1)
-
 	// Start the server
-	go func() {
-		log.Println("Starting server...")
-		err = startServer()
-		if err != nil {
-			log.Println("Failed to start server:")
-			log.Println(err)
-		}
-		quit <- os.Interrupt
-	}()
+	log.Println("Starting server...")
+	err = startServer()
+	if err != nil {
+		log.Fatalln("Failed to start server:", err.Error())
+		return
+	}
+	log.Println("Stopped server(s)")
 
-	// Graceful shutdown
-	signal.Notify(quit, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
-	<-quit
-	log.Println("Stopping...")
+	// Wait till everything is shutdown
+	waitForShutdown()
+
+	// Close DB
+	if err = closeDb(); err != nil {
+		log.Fatalln("Failed to close DB:", err.Error())
+		return
+	}
+	log.Println("Closed Database")
 
 	// Write memory profile
 	if *memprofile != "" {
 		f, err := os.Create(*memprofile)
 		if err != nil {
-			log.Fatal("could not create memory profile: ", err)
+			log.Fatalln("could not create memory profile: ", err.Error())
+			return
 		}
 		defer f.Close()
 		runtime.GC()
 		if err := pprof.WriteHeapProfile(f); err != nil {
-			log.Fatal("could not write memory profile: ", err)
+			log.Fatalln("could not write memory profile: ", err.Error())
+			return
 		}
 	}
-
-	// Close DB
-	err = closeDb()
-	if err != nil {
-		log.Fatal(err)
-		return
-	}
 }

shutdown.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+)
+
+var shutdownWg sync.WaitGroup
+
+func onShutdown(f func()) {
+	defer shutdownWg.Done()
+	shutdownWg.Add(1)
+	quit := make(chan os.Signal, 1)
+	signal.Notify(quit, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
+	<-quit
+	f()
+}
+
+func waitForShutdown() {
+	shutdownWg.Wait()
+}

tor.go

@@ -82,5 +82,13 @@ func startOnionService(h http.Handler) error {
 		ReadTimeout:  5 * time.Minute,
 		WriteTimeout: 5 * time.Minute,
 	}
-	return s.Serve(onion)
+	go onShutdown(func() {
+		toc, c := context.WithTimeout(context.Background(), 5*time.Second)
+		_ = s.Shutdown(toc)
+		c()
+	})
+	if err = s.Serve(onion); err != nil && err != http.ErrServerClosed {
+		return err
+	}
+	return nil
 }

utils.go

@@ -166,3 +166,7 @@ func dateFormat(date string, format string) string {
 	}
 	return d.Local().Format(format)
 }
+
+type stringPair struct {
+	first, second string
+}