From aa2f030b51080710dcc5a747952f457efe5d33b5 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Else Date: Sat, 25 Apr 2020 11:38:15 +0200 Subject: [PATCH] Initial commit --- .gitignore | 2 + Dockerfile | 13 ++++++ feed.go | 35 +++++++++++++++ go.mod | 10 +++++ go.sum | 22 ++++++++++ main.go | 124 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 206 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 feed.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a3d8ab2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.idea/ +/webmentionhelper \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d74fbb8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM golang:1.14-alpine as build +ADD . /app +WORKDIR /app +RUN go build + +FROM alpine:3.11 +RUN apk add --no-cache tzdata ca-certificates +COPY --from=build /app/webmentionhelper /bin/ +WORKDIR /app +VOLUME /app/storage +EXPOSE 8080 +ENV LAST_ARTICLE_DIR /app/storage +CMD ["webmentionhelper"] \ No newline at end of file diff --git a/feed.go b/feed.go new file mode 100644 index 0000000..476e141 --- /dev/null +++ b/feed.go @@ -0,0 +1,35 @@ +package main + +import ( + "encoding/json" + "errors" + "net/http" +) + +type Article struct { + Url string `json:"url"` + Modified string `json:"date_modified"` +} + +func LatestArticle(url string) (*Article, error) { + jsonFeed := &struct { + Items []Article `json:"items"` + }{} + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, errors.New("failed to create req to get json feed") + } + req.Header.Add("User-Agent", "WebmentionHelper") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, errors.New("failed to get json feed") + } + err = json.NewDecoder(resp.Body).Decode(&jsonFeed) + if err != nil { + return nil, errors.New("failed to parse json feed") + } + if len(jsonFeed.Items) < 1 { + return nil, errors.New("no articles in feed") + } + return &jsonFeed.Items[0], nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9d8ea4b --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module git.jlel.se/jlelse/webmentionhelper + +go 1.14 + +require ( + github.com/andybalholm/cascadia v1.1.0 // indirect + github.com/google/go-cmp v0.4.0 // indirect + golang.org/x/net v0.0.0-20200421231249-e086a090c8fd // indirect + willnorris.com/go/webmention v0.0.0-20200126231626-5a55fff6bf71 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..825a185 --- /dev/null +++ b/go.sum @@ -0,0 +1,22 @@ +github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd h1:QPwSajcTUrFriMF1nJ3XzgoqakqQEsnZf9LdXdi2nkI= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +willnorris.com/go/webmention v0.0.0-20200126231626-5a55fff6bf71 h1:F//bgirx4BIsJYHrqAYCZHODn0gSRej/KfueFlarXhs= +willnorris.com/go/webmention v0.0.0-20200126231626-5a55fff6bf71/go.mod h1:p+ZRAsZS2pzZ6kX3GKWYurf3WZI2ygj7VbR8NM8qwfM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..d744a3b --- /dev/null +++ b/main.go @@ -0,0 +1,124 @@ +package main + +import ( + "encoding/base64" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "path" + "strings" + + "willnorris.com/go/webmention" +) + +func main() { + lastArticleDir, lastArticleDirSet := os.LookupEnv("LAST_ARTICLE_DIR") + blacklistString, _ := os.LookupEnv("BLACKLIST") + blacklist := strings.Split(blacklistString, ",") + if lastArticleDirSet { + http.HandleFunc("/hook", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "Wrong HTTP method", http.StatusMethodNotAllowed) + return + } + + feeds, feedsPresent := r.URL.Query()["feed"] + if feedsPresent { + go func() { + for _, feed := range feeds { + fmt.Println("Check webmentions for feed:", feed) + article, err := LatestArticle(feed) + if err != nil { + fmt.Println(err.Error()) + continue + } + lastArticleUrl, lastArticleDate := lastArticle(lastArticleDir, feed) + if lastArticleUrl != article.Url || lastArticleDate != article.Modified { + err := sendWebmentions(article.Url, blacklist) + if err != nil { + fmt.Println(err.Error()) + continue + } + err = updateLastArticle(lastArticleDir, feed, article.Url, article.Modified) + if err != nil { + fmt.Println(err.Error()) + continue + } + } else { + fmt.Println("No new article") + continue + } + } + }() + } + + urls, urlsPresent := r.URL.Query()["url"] + if urlsPresent { + go func() { + for _, url := range urls { + fmt.Println("Check webmentions for url:", url) + err := sendWebmentions(url, blacklist) + if err != nil { + fmt.Println(err.Error()) + continue + } + } + }() + } + + }) + log.Fatal(http.ListenAndServe(":8080", nil)) + } else { + log.Fatal("Not configured") + } +} + +func lastArticle(dirname string, feed string) (url string, date string) { + fileContent, _ := ioutil.ReadFile(path.Join(dirname, base64.StdEncoding.EncodeToString([]byte(feed)))) + if len(string(fileContent)) == 0 { + return + } + urlDate := strings.SplitN(string(fileContent), ";", 2) + return urlDate[0], urlDate[1] +} + +func updateLastArticle(dirname string, feed string, url string, date string) error { + return ioutil.WriteFile(path.Join(dirname, base64.StdEncoding.EncodeToString([]byte(feed))), []byte(url+";"+date), 0644) +} + +func sendWebmentions(url string, blacklist []string) error { + client := webmention.New(nil) + discovered, err := client.DiscoverLinks(url, ".h-entry") + if err != nil { + return err + } + var filtered []string + allowed := func(link string) bool { + for _, black := range blacklist { + if strings.Contains(link, black) { + return false + } + } + return true + } + for _, link := range discovered { + if allowed(link) { + filtered = append(filtered, link) + } + } + for _, link := range filtered { + endpoint, err := client.DiscoverEndpoint(link) + if err != nil || len(endpoint) < 1 { + continue + } + _, err = client.SendWebmention(endpoint, url, link) + if err != nil { + log.Println("Sending webmention to " + link + " failed") + continue + } + fmt.Println("Sent webmention to " + link) + } + return nil +}