mirror of https://github.com/jlelse/GoBlog
Command "check" to check for broken external links and improve graceful shutdown
parent
0d7f615240
commit
06a1a0cdde
@ -0,0 +1,100 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
func checkAllExternalLinks() {
|
||||
allPosts, err := getPosts(&postsRequestConfig{status: statusPublished})
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
return
|
||||
}
|
||||
wg := new(sync.WaitGroup)
|
||||
linkChan := make(chan stringPair)
|
||||
client := &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
DisableKeepAlives: true,
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
responses := map[string]int{}
|
||||
rm := sync.RWMutex{}
|
||||
for i := 0; i < 20; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
wg.Add(1)
|
||||
for postLinkPair := range linkChan {
|
||||
rm.RLock()
|
||||
_, ok := responses[postLinkPair.second]
|
||||
rm.RUnlock()
|
||||
if !ok {
|
||||
req, err := http.NewRequest(http.MethodGet, postLinkPair.second, nil)
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
continue
|
||||
}
|
||||
// User-Agent from Tor
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.5")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
fmt.Println(postLinkPair.second+" ("+postLinkPair.first+"):", err.Error())
|
||||
continue
|
||||
}
|
||||
status := resp.StatusCode
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
rm.Lock()
|
||||
responses[postLinkPair.second] = status
|
||||
rm.Unlock()
|
||||
}
|
||||
rm.RLock()
|
||||
if response, ok := responses[postLinkPair.second]; ok && !checkSuccessStatus(response) {
|
||||
fmt.Println(postLinkPair.second+" ("+postLinkPair.first+"):", response)
|
||||
}
|
||||
rm.RUnlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
err = getExternalLinks(allPosts, linkChan)
|
||||
if err != nil {
|
||||
log.Println(err.Error())
|
||||
return
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func checkSuccessStatus(status int) bool {
|
||||
return status >= 200 && status < 400
|
||||
}
|
||||
|
||||
func getExternalLinks(posts []*post, linkChan chan<- stringPair) error {
|
||||
wg := new(sync.WaitGroup)
|
||||
for _, p := range posts {
|
||||
wg.Add(1)
|
||||
go func(p *post) {
|
||||
defer wg.Done()
|
||||
links, _ := allLinksFromHTML(strings.NewReader(string(p.absoluteHTML())), p.fullURL())
|
||||
for _, link := range links {
|
||||
if !strings.HasPrefix(link, appConfig.Server.PublicAddress) {
|
||||
linkChan <- stringPair{p.fullURL(), link}
|
||||
}
|
||||
}
|
||||
}(p)
|
||||
}
|
||||
wg.Wait()
|
||||
close(linkChan)
|
||||
return nil
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
var shutdownWg sync.WaitGroup
|
||||
|
||||
func onShutdown(f func()) {
|
||||
defer shutdownWg.Done()
|
||||
shutdownWg.Add(1)
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
|
||||
<-quit
|
||||
f()
|
||||
}
|
||||
|
||||
func waitForShutdown() {
|
||||
shutdownWg.Wait()
|
||||
}
|
||||
Loading…
Reference in New Issue