Command "check" to check for broken external links and improve graceful shutdown

This commit is contained in:
Jan-Lukas Else 2021-04-03 15:39:43 +02:00
parent 0d7f615240
commit 06a1a0cdde
9 changed files with 236 additions and 90 deletions

7
.vscode/tasks.json vendored
View File

@ -4,7 +4,12 @@
{
"label": "Build",
"type": "shell",
"command": "go build --tags \"libsqlite3 linux sqlite_fts5\""
"command": "go build --tags \"libsqlite3 linux sqlite_fts5\"",
"problemMatcher": [],
"group": {
"kind": "build",
"isDefault": true
}
}
]
}

100
check.go Normal file
View File

@ -0,0 +1,100 @@
package main
import (
"crypto/tls"
"fmt"
"io"
"log"
"net/http"
"strings"
"sync"
"time"
)
func checkAllExternalLinks() {
allPosts, err := getPosts(&postsRequestConfig{status: statusPublished})
if err != nil {
log.Println(err.Error())
return
}
wg := new(sync.WaitGroup)
linkChan := make(chan stringPair)
client := &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{
DisableKeepAlives: true,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
}
responses := map[string]int{}
rm := sync.RWMutex{}
for i := 0; i < 20; i++ {
go func() {
defer wg.Done()
wg.Add(1)
for postLinkPair := range linkChan {
rm.RLock()
_, ok := responses[postLinkPair.second]
rm.RUnlock()
if !ok {
req, err := http.NewRequest(http.MethodGet, postLinkPair.second, nil)
if err != nil {
fmt.Println(err.Error())
continue
}
// User-Agent from Tor
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
resp, err := client.Do(req)
if err != nil {
fmt.Println(postLinkPair.second+" ("+postLinkPair.first+"):", err.Error())
continue
}
status := resp.StatusCode
_, _ = io.Copy(io.Discard, resp.Body)
resp.Body.Close()
rm.Lock()
responses[postLinkPair.second] = status
rm.Unlock()
}
rm.RLock()
if response, ok := responses[postLinkPair.second]; ok && !checkSuccessStatus(response) {
fmt.Println(postLinkPair.second+" ("+postLinkPair.first+"):", response)
}
rm.RUnlock()
}
}()
}
err = getExternalLinks(allPosts, linkChan)
if err != nil {
log.Println(err.Error())
return
}
wg.Wait()
}
func checkSuccessStatus(status int) bool {
return status >= 200 && status < 400
}
func getExternalLinks(posts []*post, linkChan chan<- stringPair) error {
wg := new(sync.WaitGroup)
for _, p := range posts {
wg.Add(1)
go func(p *post) {
defer wg.Done()
links, _ := allLinksFromHTML(strings.NewReader(string(p.absoluteHTML())), p.fullURL())
for _, link := range links {
if !strings.HasPrefix(link, appConfig.Server.PublicAddress) {
linkChan <- stringPair{p.fullURL(), link}
}
}
}(p)
}
wg.Wait()
close(linkChan)
return nil
}

4
go.mod
View File

@ -58,9 +58,9 @@ require (
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
golang.org/x/mod v0.4.1 // indirect
golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c // indirect
golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 // indirect
golang.org/x/sys v0.0.0-20210402192133-700132347e07 // indirect
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
golang.org/x/text v0.3.6 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect

4
go.sum
View File

@ -454,8 +454,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 h1:rF3Ohx8DRyl8h2zw9qojyLHLhrJpEMgyPOImREEryf0=
golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210402192133-700132347e07 h1:4k6HsQjxj6hVMsI2Vf0yKlzt5lXxZsMW1q0zaq2k8zY=
golang.org/x/sys v0.0.0-20210402192133-700132347e07/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

40
http.go
View File

@ -17,6 +17,7 @@ import (
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
servertiming "github.com/mitchellh/go-server-timing"
"golang.org/x/net/context"
)
const (
@ -59,20 +60,19 @@ func startServer() (err error) {
finalHandler = logMiddleware(finalHandler)
}
// Create routers that don't change
err = buildStaticHandlersRouters()
if err != nil {
return
if err = buildStaticHandlersRouters(); err != nil {
return err
}
// Load router
err = reloadRouter()
if err != nil {
return
if err = reloadRouter(); err != nil {
return err
}
// Start Onion service
if appConfig.Server.Tor {
go func() {
torErr := startOnionService(finalHandler)
log.Println("Tor failed:", torErr.Error())
if err := startOnionService(finalHandler); err != nil {
log.Println("Tor failed:", err.Error())
}
}()
}
// Start server
@ -81,20 +81,30 @@ func startServer() (err error) {
ReadTimeout: 5 * time.Minute,
WriteTimeout: 5 * time.Minute,
}
go onShutdown(func() {
toc, c := context.WithTimeout(context.Background(), 5*time.Second)
_ = s.Shutdown(toc)
c()
})
if appConfig.Server.PublicHTTPS {
// Configure
certmagic.Default.Storage = &certmagic.FileStorage{Path: "data/https"}
certmagic.DefaultACME.Email = appConfig.Server.LetsEncryptMail
certmagic.DefaultACME.CA = certmagic.LetsEncryptProductionCA
// Start HTTP server for TLS verification and redirect
// Start HTTP server for redirects
httpServer := &http.Server{
Addr: ":http",
Handler: http.HandlerFunc(redirectToHttps),
ReadTimeout: 5 * time.Minute,
WriteTimeout: 5 * time.Minute,
}
go onShutdown(func() {
toc, c := context.WithTimeout(context.Background(), 5*time.Second)
_ = httpServer.Shutdown(toc)
c()
})
go func() {
if err := httpServer.ListenAndServe(); err != nil {
if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Println("Failed to start HTTP server:", err.Error())
}
}()
@ -108,12 +118,16 @@ func startServer() (err error) {
if e != nil {
return e
}
err = s.Serve(listener)
if err = s.Serve(listener); err != nil && err != http.ErrServerClosed {
return err
}
} else {
s.Addr = ":" + strconv.Itoa(appConfig.Server.Port)
err = s.ListenAndServe()
if err = s.ListenAndServe(); err != nil && err != http.ErrServerClosed {
return err
}
}
return
return nil
}
func redirectToHttps(w http.ResponseWriter, r *http.Request) {

134
main.go
View File

@ -4,10 +4,8 @@ import (
"flag"
"log"
"os"
"os/signal"
"runtime"
"runtime/pprof"
"syscall"
"github.com/pquerna/otp/totp"
)
@ -16,6 +14,8 @@ var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to `file`")
var memprofile = flag.String("memprofile", "", "write memory profile to `file`")
func main() {
var err error
// Init CPU profiling
flag.Parse()
if *cpuprofile != "" {
@ -32,25 +32,22 @@ func main() {
// Initialize config
log.Println("Initialize configuration...")
err := initConfig()
if err != nil {
log.Fatal(err)
if err = initConfig(); err != nil {
log.Fatalln("Failed to init config:", err.Error())
}
// Small tools
if len(os.Args) >= 2 {
if os.Args[1] == "totp-secret" {
key, err := totp.Generate(totp.GenerateOpts{
Issuer: appConfig.Server.PublicAddress,
AccountName: appConfig.User.Nick,
})
if err != nil {
log.Fatal(err.Error())
return
}
log.Println("TOTP-Secret:", key.Secret())
// Small tools before init
if len(os.Args) >= 2 && os.Args[1] == "totp-secret" {
key, err := totp.Generate(totp.GenerateOpts{
Issuer: appConfig.Server.PublicAddress,
AccountName: appConfig.User.Nick,
})
if err != nil {
log.Fatalln(err.Error())
return
}
log.Println("TOTP-Secret:", key.Secret())
return
}
// Init regular garbage collection
@ -58,54 +55,54 @@ func main() {
// Execute pre-start hooks
preStartHooks()
// Initialize everything else
// Initialize database and markdown
log.Println("Initialize database...")
err = initDatabase()
if err != nil {
log.Fatal(err)
if err = initDatabase(); err != nil {
log.Fatalln("Failed to init database:", err.Error())
return
}
log.Println("Initialize server components...")
initMinify()
initMarkdown()
err = initTemplateAssets() // Needs minify
if err != nil {
log.Fatal(err)
// Link check tool after init of markdown
if len(os.Args) >= 2 && os.Args[1] == "check" {
checkAllExternalLinks()
return
}
err = initTemplateStrings()
if err != nil {
log.Fatal(err)
// More initializations
initMinify()
if err = initTemplateAssets(); err != nil { // Needs minify
log.Fatalln("Failed to init template assets:", err.Error())
return
}
err = initRendering() // Needs assets
if err != nil {
log.Fatal(err)
if err = initTemplateStrings(); err != nil {
log.Fatalln("Failed to init template translations:", err.Error())
return
}
err = initCache()
if err != nil {
log.Fatal(err)
if err = initRendering(); err != nil { // Needs assets and minify
log.Fatalln("Failed to init HTML rendering:", err.Error())
return
}
err = initRegexRedirects()
if err != nil {
log.Fatal(err)
if err = initCache(); err != nil {
log.Fatalln("Failed to init HTTP cache:", err.Error())
return
}
err = initHTTPLog()
if err != nil {
log.Fatal(err)
if err = initRegexRedirects(); err != nil {
log.Fatalln("Failed to init redirects:", err.Error())
return
}
err = initActivityPub()
if err != nil {
log.Fatal(err)
if err = initHTTPLog(); err != nil {
log.Fatal("Failed to init HTTP logging:", err.Error())
return
}
err = initWebmention()
if err != nil {
log.Fatal(err)
if err = initActivityPub(); err != nil {
log.Fatalln("Failed to init ActivityPub:", err.Error())
return
}
if err = initWebmention(); err != nil {
log.Fatalln("Failed to init webmention support:", err.Error())
return
}
initTelegram()
@ -113,42 +110,37 @@ func main() {
// Start cron hooks
startHourlyHooks()
// Prepare graceful shutdown
quit := make(chan os.Signal, 1)
// Start the server
go func() {
log.Println("Starting server...")
err = startServer()
if err != nil {
log.Println("Failed to start server:")
log.Println(err)
}
quit <- os.Interrupt
}()
log.Println("Starting server...")
err = startServer()
if err != nil {
log.Fatalln("Failed to start server:", err.Error())
return
}
log.Println("Stopped server(s)")
// Graceful shutdown
signal.Notify(quit, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
<-quit
log.Println("Stopping...")
// Wait till everything is shutdown
waitForShutdown()
// Close DB
if err = closeDb(); err != nil {
log.Fatalln("Failed to close DB:", err.Error())
return
}
log.Println("Closed Database")
// Write memory profile
if *memprofile != "" {
f, err := os.Create(*memprofile)
if err != nil {
log.Fatal("could not create memory profile: ", err)
log.Fatalln("could not create memory profile: ", err.Error())
return
}
defer f.Close()
runtime.GC()
if err := pprof.WriteHeapProfile(f); err != nil {
log.Fatal("could not write memory profile: ", err)
log.Fatalln("could not write memory profile: ", err.Error())
return
}
}
// Close DB
err = closeDb()
if err != nil {
log.Fatal(err)
return
}
}

23
shutdown.go Normal file
View File

@ -0,0 +1,23 @@
package main
import (
"os"
"os/signal"
"sync"
"syscall"
)
var shutdownWg sync.WaitGroup
func onShutdown(f func()) {
defer shutdownWg.Done()
shutdownWg.Add(1)
quit := make(chan os.Signal, 1)
signal.Notify(quit, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
<-quit
f()
}
func waitForShutdown() {
shutdownWg.Wait()
}

10
tor.go
View File

@ -82,5 +82,13 @@ func startOnionService(h http.Handler) error {
ReadTimeout: 5 * time.Minute,
WriteTimeout: 5 * time.Minute,
}
return s.Serve(onion)
go onShutdown(func() {
toc, c := context.WithTimeout(context.Background(), 5*time.Second)
_ = s.Shutdown(toc)
c()
})
if err = s.Serve(onion); err != nil && err != http.ErrServerClosed {
return err
}
return nil
}

View File

@ -166,3 +166,7 @@ func dateFormat(date string, format string) string {
}
return d.Local().Format(format)
}
type stringPair struct {
first, second string
}