mirror of https://github.com/jlelse/GoBlog
Command "check" to check for broken external links and improve graceful shutdown
This commit is contained in:
parent
0d7f615240
commit
06a1a0cdde
|
@ -4,7 +4,12 @@
|
|||
{
|
||||
"label": "Build",
|
||||
"type": "shell",
|
||||
"command": "go build --tags \"libsqlite3 linux sqlite_fts5\""
|
||||
"command": "go build --tags \"libsqlite3 linux sqlite_fts5\"",
|
||||
"problemMatcher": [],
|
||||
"group": {
|
||||
"kind": "build",
|
||||
"isDefault": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
|
@ -0,0 +1,100 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
func checkAllExternalLinks() {
|
||||
allPosts, err := getPosts(&postsRequestConfig{status: statusPublished})
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
return
|
||||
}
|
||||
wg := new(sync.WaitGroup)
|
||||
linkChan := make(chan stringPair)
|
||||
client := &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
DisableKeepAlives: true,
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
responses := map[string]int{}
|
||||
rm := sync.RWMutex{}
|
||||
for i := 0; i < 20; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
wg.Add(1)
|
||||
for postLinkPair := range linkChan {
|
||||
rm.RLock()
|
||||
_, ok := responses[postLinkPair.second]
|
||||
rm.RUnlock()
|
||||
if !ok {
|
||||
req, err := http.NewRequest(http.MethodGet, postLinkPair.second, nil)
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
continue
|
||||
}
|
||||
// User-Agent from Tor
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
fmt.Println(postLinkPair.second+" ("+postLinkPair.first+"):", err.Error())
|
||||
continue
|
||||
}
|
||||
status := resp.StatusCode
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
rm.Lock()
|
||||
responses[postLinkPair.second] = status
|
||||
rm.Unlock()
|
||||
}
|
||||
rm.RLock()
|
||||
if response, ok := responses[postLinkPair.second]; ok && !checkSuccessStatus(response) {
|
||||
fmt.Println(postLinkPair.second+" ("+postLinkPair.first+"):", response)
|
||||
}
|
||||
rm.RUnlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
err = getExternalLinks(allPosts, linkChan)
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
return
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func checkSuccessStatus(status int) bool {
|
||||
return status >= 200 && status < 400
|
||||
}
|
||||
|
||||
func getExternalLinks(posts []*post, linkChan chan<- stringPair) error {
|
||||
wg := new(sync.WaitGroup)
|
||||
for _, p := range posts {
|
||||
wg.Add(1)
|
||||
go func(p *post) {
|
||||
defer wg.Done()
|
||||
links, _ := allLinksFromHTML(strings.NewReader(string(p.absoluteHTML())), p.fullURL())
|
||||
for _, link := range links {
|
||||
if !strings.HasPrefix(link, appConfig.Server.PublicAddress) {
|
||||
linkChan <- stringPair{p.fullURL(), link}
|
||||
}
|
||||
}
|
||||
}(p)
|
||||
}
|
||||
wg.Wait()
|
||||
close(linkChan)
|
||||
return nil
|
||||
}
|
4
go.mod
4
go.mod
|
@ -58,9 +58,9 @@ require (
|
|||
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect
|
||||
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
|
||||
golang.org/x/mod v0.4.1 // indirect
|
||||
golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c // indirect
|
||||
golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||
golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 // indirect
|
||||
golang.org/x/sys v0.0.0-20210402192133-700132347e07 // indirect
|
||||
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
|
||||
golang.org/x/text v0.3.6 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
|
|
4
go.sum
4
go.sum
|
@ -454,8 +454,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 h1:rF3Ohx8DRyl8h2zw9qojyLHLhrJpEMgyPOImREEryf0=
|
||||
golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210402192133-700132347e07 h1:4k6HsQjxj6hVMsI2Vf0yKlzt5lXxZsMW1q0zaq2k8zY=
|
||||
golang.org/x/sys v0.0.0-20210402192133-700132347e07/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=
|
||||
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
|
|
40
http.go
40
http.go
|
@ -17,6 +17,7 @@ import (
|
|||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
servertiming "github.com/mitchellh/go-server-timing"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
const (
|
||||
|
@ -59,20 +60,19 @@ func startServer() (err error) {
|
|||
finalHandler = logMiddleware(finalHandler)
|
||||
}
|
||||
// Create routers that don't change
|
||||
err = buildStaticHandlersRouters()
|
||||
if err != nil {
|
||||
return
|
||||
if err = buildStaticHandlersRouters(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Load router
|
||||
err = reloadRouter()
|
||||
if err != nil {
|
||||
return
|
||||
if err = reloadRouter(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Start Onion service
|
||||
if appConfig.Server.Tor {
|
||||
go func() {
|
||||
torErr := startOnionService(finalHandler)
|
||||
log.Println("Tor failed:", torErr.Error())
|
||||
if err := startOnionService(finalHandler); err != nil {
|
||||
log.Println("Tor failed:", err.Error())
|
||||
}
|
||||
}()
|
||||
}
|
||||
// Start server
|
||||
|
@ -81,20 +81,30 @@ func startServer() (err error) {
|
|||
ReadTimeout: 5 * time.Minute,
|
||||
WriteTimeout: 5 * time.Minute,
|
||||
}
|
||||
go onShutdown(func() {
|
||||
toc, c := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
_ = s.Shutdown(toc)
|
||||
c()
|
||||
})
|
||||
if appConfig.Server.PublicHTTPS {
|
||||
// Configure
|
||||
certmagic.Default.Storage = &certmagic.FileStorage{Path: "data/https"}
|
||||
certmagic.DefaultACME.Email = appConfig.Server.LetsEncryptMail
|
||||
certmagic.DefaultACME.CA = certmagic.LetsEncryptProductionCA
|
||||
// Start HTTP server for TLS verification and redirect
|
||||
// Start HTTP server for redirects
|
||||
httpServer := &http.Server{
|
||||
Addr: ":http",
|
||||
Handler: http.HandlerFunc(redirectToHttps),
|
||||
ReadTimeout: 5 * time.Minute,
|
||||
WriteTimeout: 5 * time.Minute,
|
||||
}
|
||||
go onShutdown(func() {
|
||||
toc, c := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
_ = httpServer.Shutdown(toc)
|
||||
c()
|
||||
})
|
||||
go func() {
|
||||
if err := httpServer.ListenAndServe(); err != nil {
|
||||
if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
log.Println("Failed to start HTTP server:", err.Error())
|
||||
}
|
||||
}()
|
||||
|
@ -108,12 +118,16 @@ func startServer() (err error) {
|
|||
if e != nil {
|
||||
return e
|
||||
}
|
||||
err = s.Serve(listener)
|
||||
if err = s.Serve(listener); err != nil && err != http.ErrServerClosed {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
s.Addr = ":" + strconv.Itoa(appConfig.Server.Port)
|
||||
err = s.ListenAndServe()
|
||||
if err = s.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
func redirectToHttps(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
134
main.go
134
main.go
|
@ -4,10 +4,8 @@ import (
|
|||
"flag"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"runtime/pprof"
|
||||
"syscall"
|
||||
|
||||
"github.com/pquerna/otp/totp"
|
||||
)
|
||||
|
@ -16,6 +14,8 @@ var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to `file`")
|
|||
var memprofile = flag.String("memprofile", "", "write memory profile to `file`")
|
||||
|
||||
func main() {
|
||||
var err error
|
||||
|
||||
// Init CPU profiling
|
||||
flag.Parse()
|
||||
if *cpuprofile != "" {
|
||||
|
@ -32,25 +32,22 @@ func main() {
|
|||
|
||||
// Initialize config
|
||||
log.Println("Initialize configuration...")
|
||||
err := initConfig()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
if err = initConfig(); err != nil {
|
||||
log.Fatalln("Failed to init config:", err.Error())
|
||||
}
|
||||
|
||||
// Small tools
|
||||
if len(os.Args) >= 2 {
|
||||
if os.Args[1] == "totp-secret" {
|
||||
key, err := totp.Generate(totp.GenerateOpts{
|
||||
Issuer: appConfig.Server.PublicAddress,
|
||||
AccountName: appConfig.User.Nick,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err.Error())
|
||||
return
|
||||
}
|
||||
log.Println("TOTP-Secret:", key.Secret())
|
||||
// Small tools before init
|
||||
if len(os.Args) >= 2 && os.Args[1] == "totp-secret" {
|
||||
key, err := totp.Generate(totp.GenerateOpts{
|
||||
Issuer: appConfig.Server.PublicAddress,
|
||||
AccountName: appConfig.User.Nick,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatalln(err.Error())
|
||||
return
|
||||
}
|
||||
log.Println("TOTP-Secret:", key.Secret())
|
||||
return
|
||||
}
|
||||
|
||||
// Init regular garbage collection
|
||||
|
@ -58,54 +55,54 @@ func main() {
|
|||
|
||||
// Execute pre-start hooks
|
||||
preStartHooks()
|
||||
// Initialize everything else
|
||||
|
||||
// Initialize database and markdown
|
||||
log.Println("Initialize database...")
|
||||
err = initDatabase()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
if err = initDatabase(); err != nil {
|
||||
log.Fatalln("Failed to init database:", err.Error())
|
||||
return
|
||||
}
|
||||
log.Println("Initialize server components...")
|
||||
initMinify()
|
||||
initMarkdown()
|
||||
err = initTemplateAssets() // Needs minify
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
|
||||
// Link check tool after init of markdown
|
||||
if len(os.Args) >= 2 && os.Args[1] == "check" {
|
||||
checkAllExternalLinks()
|
||||
return
|
||||
}
|
||||
err = initTemplateStrings()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
|
||||
// More initializations
|
||||
initMinify()
|
||||
if err = initTemplateAssets(); err != nil { // Needs minify
|
||||
log.Fatalln("Failed to init template assets:", err.Error())
|
||||
return
|
||||
}
|
||||
err = initRendering() // Needs assets
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
if err = initTemplateStrings(); err != nil {
|
||||
log.Fatalln("Failed to init template translations:", err.Error())
|
||||
return
|
||||
}
|
||||
err = initCache()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
if err = initRendering(); err != nil { // Needs assets and minify
|
||||
log.Fatalln("Failed to init HTML rendering:", err.Error())
|
||||
return
|
||||
}
|
||||
err = initRegexRedirects()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
if err = initCache(); err != nil {
|
||||
log.Fatalln("Failed to init HTTP cache:", err.Error())
|
||||
return
|
||||
}
|
||||
err = initHTTPLog()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
if err = initRegexRedirects(); err != nil {
|
||||
log.Fatalln("Failed to init redirects:", err.Error())
|
||||
return
|
||||
}
|
||||
err = initActivityPub()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
if err = initHTTPLog(); err != nil {
|
||||
log.Fatal("Failed to init HTTP logging:", err.Error())
|
||||
return
|
||||
}
|
||||
err = initWebmention()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
if err = initActivityPub(); err != nil {
|
||||
log.Fatalln("Failed to init ActivityPub:", err.Error())
|
||||
return
|
||||
}
|
||||
if err = initWebmention(); err != nil {
|
||||
log.Fatalln("Failed to init webmention support:", err.Error())
|
||||
return
|
||||
}
|
||||
initTelegram()
|
||||
|
@ -113,42 +110,37 @@ func main() {
|
|||
// Start cron hooks
|
||||
startHourlyHooks()
|
||||
|
||||
// Prepare graceful shutdown
|
||||
quit := make(chan os.Signal, 1)
|
||||
|
||||
// Start the server
|
||||
go func() {
|
||||
log.Println("Starting server...")
|
||||
err = startServer()
|
||||
if err != nil {
|
||||
log.Println("Failed to start server:")
|
||||
log.Println(err)
|
||||
}
|
||||
quit <- os.Interrupt
|
||||
}()
|
||||
log.Println("Starting server...")
|
||||
err = startServer()
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to start server:", err.Error())
|
||||
return
|
||||
}
|
||||
log.Println("Stopped server(s)")
|
||||
|
||||
// Graceful shutdown
|
||||
signal.Notify(quit, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
|
||||
<-quit
|
||||
log.Println("Stopping...")
|
||||
// Wait till everything is shutdown
|
||||
waitForShutdown()
|
||||
|
||||
// Close DB
|
||||
if err = closeDb(); err != nil {
|
||||
log.Fatalln("Failed to close DB:", err.Error())
|
||||
return
|
||||
}
|
||||
log.Println("Closed Database")
|
||||
|
||||
// Write memory profile
|
||||
if *memprofile != "" {
|
||||
f, err := os.Create(*memprofile)
|
||||
if err != nil {
|
||||
log.Fatal("could not create memory profile: ", err)
|
||||
log.Fatalln("could not create memory profile: ", err.Error())
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
runtime.GC()
|
||||
if err := pprof.WriteHeapProfile(f); err != nil {
|
||||
log.Fatal("could not write memory profile: ", err)
|
||||
log.Fatalln("could not write memory profile: ", err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Close DB
|
||||
err = closeDb()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,23 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
var shutdownWg sync.WaitGroup
|
||||
|
||||
func onShutdown(f func()) {
|
||||
defer shutdownWg.Done()
|
||||
shutdownWg.Add(1)
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
|
||||
<-quit
|
||||
f()
|
||||
}
|
||||
|
||||
func waitForShutdown() {
|
||||
shutdownWg.Wait()
|
||||
}
|
10
tor.go
10
tor.go
|
@ -82,5 +82,13 @@ func startOnionService(h http.Handler) error {
|
|||
ReadTimeout: 5 * time.Minute,
|
||||
WriteTimeout: 5 * time.Minute,
|
||||
}
|
||||
return s.Serve(onion)
|
||||
go onShutdown(func() {
|
||||
toc, c := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
_ = s.Shutdown(toc)
|
||||
c()
|
||||
})
|
||||
if err = s.Serve(onion); err != nil && err != http.ErrServerClosed {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue