Initial commit
This commit is contained in:
commit
aa2f030b51
|
@ -0,0 +1,2 @@
|
||||||
|
.idea/
|
||||||
|
/webmentionhelper
|
|
@ -0,0 +1,13 @@
|
||||||
|
FROM golang:1.14-alpine as build
|
||||||
|
ADD . /app
|
||||||
|
WORKDIR /app
|
||||||
|
RUN go build
|
||||||
|
|
||||||
|
FROM alpine:3.11
|
||||||
|
RUN apk add --no-cache tzdata ca-certificates
|
||||||
|
COPY --from=build /app/webmentionhelper /bin/
|
||||||
|
WORKDIR /app
|
||||||
|
VOLUME /app/storage
|
||||||
|
EXPOSE 8080
|
||||||
|
ENV LAST_ARTICLE_DIR /app/storage
|
||||||
|
CMD ["webmentionhelper"]
|
|
@ -0,0 +1,35 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Article struct {
|
||||||
|
Url string `json:"url"`
|
||||||
|
Modified string `json:"date_modified"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func LatestArticle(url string) (*Article, error) {
|
||||||
|
jsonFeed := &struct {
|
||||||
|
Items []Article `json:"items"`
|
||||||
|
}{}
|
||||||
|
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New("failed to create req to get json feed")
|
||||||
|
}
|
||||||
|
req.Header.Add("User-Agent", "WebmentionHelper")
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New("failed to get json feed")
|
||||||
|
}
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(&jsonFeed)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New("failed to parse json feed")
|
||||||
|
}
|
||||||
|
if len(jsonFeed.Items) < 1 {
|
||||||
|
return nil, errors.New("no articles in feed")
|
||||||
|
}
|
||||||
|
return &jsonFeed.Items[0], nil
|
||||||
|
}
|
|
@ -0,0 +1,10 @@
|
||||||
|
module git.jlel.se/jlelse/webmentionhelper
|
||||||
|
|
||||||
|
go 1.14
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/andybalholm/cascadia v1.1.0 // indirect
|
||||||
|
github.com/google/go-cmp v0.4.0 // indirect
|
||||||
|
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd // indirect
|
||||||
|
willnorris.com/go/webmention v0.0.0-20200126231626-5a55fff6bf71
|
||||||
|
)
|
|
@ -0,0 +1,22 @@
|
||||||
|
github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
|
||||||
|
github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||||
|
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
|
||||||
|
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||||
|
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
|
||||||
|
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||||
|
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
|
||||||
|
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
|
||||||
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI=
|
||||||
|
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||||
|
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd h1:QPwSajcTUrFriMF1nJ3XzgoqakqQEsnZf9LdXdi2nkI=
|
||||||
|
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||||
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||||
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
willnorris.com/go/webmention v0.0.0-20200126231626-5a55fff6bf71 h1:F//bgirx4BIsJYHrqAYCZHODn0gSRej/KfueFlarXhs=
|
||||||
|
willnorris.com/go/webmention v0.0.0-20200126231626-5a55fff6bf71/go.mod h1:p+ZRAsZS2pzZ6kX3GKWYurf3WZI2ygj7VbR8NM8qwfM=
|
|
@ -0,0 +1,124 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"willnorris.com/go/webmention"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
lastArticleDir, lastArticleDirSet := os.LookupEnv("LAST_ARTICLE_DIR")
|
||||||
|
blacklistString, _ := os.LookupEnv("BLACKLIST")
|
||||||
|
blacklist := strings.Split(blacklistString, ",")
|
||||||
|
if lastArticleDirSet {
|
||||||
|
http.HandleFunc("/hook", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodPost {
|
||||||
|
http.Error(w, "Wrong HTTP method", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
feeds, feedsPresent := r.URL.Query()["feed"]
|
||||||
|
if feedsPresent {
|
||||||
|
go func() {
|
||||||
|
for _, feed := range feeds {
|
||||||
|
fmt.Println("Check webmentions for feed:", feed)
|
||||||
|
article, err := LatestArticle(feed)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println(err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
lastArticleUrl, lastArticleDate := lastArticle(lastArticleDir, feed)
|
||||||
|
if lastArticleUrl != article.Url || lastArticleDate != article.Modified {
|
||||||
|
err := sendWebmentions(article.Url, blacklist)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println(err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
err = updateLastArticle(lastArticleDir, feed, article.Url, article.Modified)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println(err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fmt.Println("No new article")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
urls, urlsPresent := r.URL.Query()["url"]
|
||||||
|
if urlsPresent {
|
||||||
|
go func() {
|
||||||
|
for _, url := range urls {
|
||||||
|
fmt.Println("Check webmentions for url:", url)
|
||||||
|
err := sendWebmentions(url, blacklist)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println(err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
})
|
||||||
|
log.Fatal(http.ListenAndServe(":8080", nil))
|
||||||
|
} else {
|
||||||
|
log.Fatal("Not configured")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func lastArticle(dirname string, feed string) (url string, date string) {
|
||||||
|
fileContent, _ := ioutil.ReadFile(path.Join(dirname, base64.StdEncoding.EncodeToString([]byte(feed))))
|
||||||
|
if len(string(fileContent)) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
urlDate := strings.SplitN(string(fileContent), ";", 2)
|
||||||
|
return urlDate[0], urlDate[1]
|
||||||
|
}
|
||||||
|
|
||||||
|
func updateLastArticle(dirname string, feed string, url string, date string) error {
|
||||||
|
return ioutil.WriteFile(path.Join(dirname, base64.StdEncoding.EncodeToString([]byte(feed))), []byte(url+";"+date), 0644)
|
||||||
|
}
|
||||||
|
|
||||||
|
func sendWebmentions(url string, blacklist []string) error {
|
||||||
|
client := webmention.New(nil)
|
||||||
|
discovered, err := client.DiscoverLinks(url, ".h-entry")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var filtered []string
|
||||||
|
allowed := func(link string) bool {
|
||||||
|
for _, black := range blacklist {
|
||||||
|
if strings.Contains(link, black) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, link := range discovered {
|
||||||
|
if allowed(link) {
|
||||||
|
filtered = append(filtered, link)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, link := range filtered {
|
||||||
|
endpoint, err := client.DiscoverEndpoint(link)
|
||||||
|
if err != nil || len(endpoint) < 1 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_, err = client.SendWebmention(endpoint, url, link)
|
||||||
|
if err != nil {
|
||||||
|
log.Println("Sending webmention to " + link + " failed")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Println("Sent webmention to " + link)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
Reference in New Issue