From 23f8e8aaad678d107f71dc8d6788f25b4651c462 Mon Sep 17 00:00:00 2001 From: Jan-Lukas Else Date: Tue, 4 Feb 2020 21:25:57 +0100 Subject: [PATCH] Init --- .gitignore | 5 + Dockerfile | 12 +++ actor.go | 250 +++++++++++++++++++++++++++++++++++++++++++++++++ config.ini | 2 + feed.go | 33 +++++++ go.mod | 11 +++ go.sum | 24 +++++ http.go | 93 ++++++++++++++++++ main.go | 41 ++++++++ remoteActor.go | 61 ++++++++++++ setup.go | 72 ++++++++++++++ util.go | 18 ++++ 12 files changed, 622 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 actor.go create mode 100644 config.ini create mode 100644 feed.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 http.go create mode 100644 main.go create mode 100644 remoteActor.go create mode 100644 setup.go create mode 100644 util.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a2a9842 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +storage +jsonpub +.idea +private.pem +public.pem \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1c944ae --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.13-alpine as build +ADD . /app +WORKDIR /app +RUN go build + +FROM alpine:3.11 +RUN apk add --no-cache tzdata ca-certificates +COPY --from=build /app/jsonpub /bin/ +WORKDIR /app +VOLUME /app/storage +EXPOSE 8081 +CMD ["jsonpub"] \ No newline at end of file diff --git a/actor.go b/actor.go new file mode 100644 index 0000000..2f84fc9 --- /dev/null +++ b/actor.go @@ -0,0 +1,250 @@ +package main + +import ( + "bytes" + "crypto" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "github.com/dchest/uniuri" + "github.com/go-fed/httpsig" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +type Actor struct { + Name, iri, feed string + followers map[string]interface{} + sentItems map[string]bool + privateKey crypto.PrivateKey + publicKeyID string +} + +type ActorToSave struct { + Name string + Followers map[string]interface{} + SentItems map[string]bool +} + +func GetActor(name, iri, feed, pk string) (Actor, error) { + // make sure users can't read hard drive + if strings.ContainsAny(name, "./ ") { + return Actor{}, errors.New("illegal characters in actor name") + } + // populate actor + pkfile, err := ioutil.ReadFile(pk) + if err != nil { + return Actor{}, errors.New("failed to read private key file") + } + privateKeyDecoded, rest := pem.Decode(pkfile) + if privateKeyDecoded == nil { + fmt.Println(rest) + return Actor{}, errors.New("failed to decode the private key PEM") + } + privateKey, err := x509.ParsePKCS1PrivateKey(privateKeyDecoded.Bytes) + if err != nil { + return Actor{}, errors.New("failed to parse the private key") + } + actor := Actor{ + Name: name, + iri: iri, + feed: feed, + followers: make(map[string]interface{}), + sentItems: make(map[string]bool), + privateKey: privateKey, + publicKeyID: iri + "#main-key", + } + jsonFile := storage + slash + "actors" + slash + name + slash + name + ".json" + fileHandle, err := os.Open(jsonFile) + if os.IsNotExist(err) { + // File doesn't exist, maybe it's a new actor + return actor, nil + } + byteValue, err := ioutil.ReadAll(fileHandle) + if err != nil { + // Ignore error, but return actor + return actor, nil + } + savedActor := ActorToSave{} + err = json.Unmarshal(byteValue, &savedActor) + if err != nil { + // Ignore error, but return actor + return actor, nil + } + if savedActor.Followers != nil { + actor.followers = savedActor.Followers + } + if savedActor.SentItems != nil { + actor.sentItems = savedActor.SentItems + } + return actor, nil +} + +// save the actor to file +func (a *Actor) save() error { + // check if directory already exists + dir := storage + slash + "actors" + slash + a.Name + if _, err := os.Stat(dir); os.IsNotExist(err) { + _ = os.MkdirAll(dir, 0755) + } + actorToSave := ActorToSave{ + Name: a.Name, + Followers: a.followers, + SentItems: a.sentItems, + } + actorJSON, err := json.MarshalIndent(actorToSave, "", "\t") + if err != nil { + return err + } + err = ioutil.WriteFile(storage+slash+"actors"+slash+a.Name+slash+a.Name+".json", actorJSON, 0644) + if err != nil { + return err + } + return nil +} + +func (a *Actor) PostArticle(url string) { + create := make(map[string]interface{}) + note := make(map[string]interface{}) + create["@context"] = context() + create["actor"] = a.iri + create["id"] = url + create["published"] = note["published"] + create["type"] = "Create" + article := make(map[string]interface{}) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return + } + req.Header.Add("User-Agent", fmt.Sprintf("%s %s", libName, version)) + req.Header.Add("Accept", ContentTypeAs2) + resp, err := client.Do(req) + if err != nil || isSuccess(resp.StatusCode) { + fmt.Println("Failed to fetch article") + return + } + respBody, err := ioutil.ReadAll(resp.Body) + defer func() { _ = resp.Body.Close() }() + if err != nil { + fmt.Println("Failed to read response from fetched article") + return + } + err = json.Unmarshal(respBody, &article) + if err != nil { + fmt.Println("Failed to unmarshal fetched article") + return + } + create["object"] = article + go func() { _ = a.sendToFollowers(create) }() +} + +// signedHTTPPost performs an HTTP post on behalf of Actor with the +// request-target, date, host and digest headers signed +// with the actor's private key. +func (a *Actor) signedHTTPPost(content map[string]interface{}, to string) (err error) { + b, err := json.Marshal(content) + if err != nil { + return + } + postSigner, _, _ := httpsig.NewSigner([]httpsig.Algorithm{httpsig.RSA_SHA256}, "SHA-256", []string{"(request-target)", "date", "host", "digest"}, httpsig.Signature) + byteCopy := make([]byte, len(b)) + copy(byteCopy, b) + buf := bytes.NewBuffer(byteCopy) + req, err := http.NewRequest(http.MethodPost, to, buf) + if err != nil { + return + } + iri, err := url.Parse(to) + if err != nil { + return err + } + req.Header.Add("Accept-Charset", "utf-8") + req.Header.Add("Date", time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT") + req.Header.Add("User-Agent", fmt.Sprintf("%s %s", libName, version)) + req.Header.Add("Host", iri.Host) + req.Header.Add("Accept", "application/activity+json; charset=utf-8") + req.Header.Add("Content-Type", "application/activity+json; charset=utf-8") + err = postSigner.SignRequest(a.privateKey, a.publicKeyID, req, byteCopy) + if err != nil { + return + } + resp, err := client.Do(req) + if err != nil { + return + } + if !isSuccess(resp.StatusCode) { + err = errors.New("post request failed") + return + } + return +} + +// NewFollower records a new follower to the actor file +func (a *Actor) NewFollower(iri string, inbox string) error { + a.followers[iri] = inbox + return a.save() +} + +func (a *Actor) RemoveFollower(iri string) error { + delete(a.followers, iri) + return a.save() +} + +// batchSend sends a batch of http posts to a list of recipients +func (a *Actor) batchSend(activity map[string]interface{}, recipients []string) (err error) { + for _, v := range recipients { + _ = a.signedHTTPPost(activity, v) + } + return +} + +// send to followers sends a batch of http posts to each one of the followers +func (a *Actor) sendToFollowers(activity map[string]interface{}) (err error) { + recipients := make([]string, len(a.followers)) + i := 0 + for _, inbox := range a.followers { + recipients[i] = inbox.(string) + i++ + } + _ = a.batchSend(activity, recipients) + return +} + +func (a *Actor) newID() (hash string, url string) { + hash = uniuri.New() + return hash, a.iri + hash +} + +// Accept a follow request +func (a *Actor) Accept(follow map[string]interface{}) { + // it's a follow, write it down + newFollower := follow["actor"].(string) + // check we aren't following ourselves + if newFollower == follow["object"] { + // actor and object are equal + return + } + follower, err := NewRemoteActor(follow["actor"].(string)) + if err != nil { + // Couldn't retrieve remote actor info + return + } + // Add or update follower + _ = a.NewFollower(newFollower, follower.inbox) + // remove @context from the inner activity + delete(follow, "@context") + accept := make(map[string]interface{}) + accept["@context"] = "https://www.w3.org/ns/activitystreams" + accept["to"] = follow["actor"] + _, accept["id"] = a.newID() + accept["actor"] = a.iri + accept["object"] = follow + accept["type"] = "Accept" + go func() { _ = a.signedHTTPPost(accept, follower.inbox) }() +} diff --git a/config.ini b/config.ini new file mode 100644 index 0000000..bfbbe2b --- /dev/null +++ b/config.ini @@ -0,0 +1,2 @@ +[general] +baseURL = https://jlelse.blog \ No newline at end of file diff --git a/feed.go b/feed.go new file mode 100644 index 0000000..49865b7 --- /dev/null +++ b/feed.go @@ -0,0 +1,33 @@ +package main + +import ( + "encoding/json" + "errors" + "io/ioutil" +) + +func allFeedItems(url string) ([]string, error) { + jsonFeed := &struct { + Items []struct { + Url string `json:"url"` + } `json:"items"` + }{} + resp, err := client.Get(url) + if err != nil { + return nil, errors.New("failed to get json feed") + } + defer func() { _ = resp.Body.Close() }() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.New("failed to read json feed") + } + err = json.Unmarshal(body, &jsonFeed) + if err != nil { + return nil, errors.New("failed to parse json feed") + } + var allUrls []string + for _, item := range jsonFeed.Items { + allUrls = append(allUrls, item.Url) + } + return allUrls, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6fcfd8f --- /dev/null +++ b/go.mod @@ -0,0 +1,11 @@ +module codeberg.org/jlelse/jsonpub + +go 1.13 + +require ( + github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 + github.com/go-fed/httpsig v0.1.1-0.20190924171022-f4c36041199d + github.com/gorilla/mux v1.7.3 + golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 // indirect + willnorris.com/go/webmention v0.0.0-20200126231626-5a55fff6bf71 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..b3f75f6 --- /dev/null +++ b/go.sum @@ -0,0 +1,24 @@ +github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 h1:74lLNRzvsdIlkTgfDSMuaPjBr4cf6k7pwQQANm/yLKU= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= +github.com/go-fed/httpsig v0.1.1-0.20190924171022-f4c36041199d h1:+uoOvOnNDgsYbWtAij4xP6Rgir3eJGjocFPxBJETU/U= +github.com/go-fed/httpsig v0.1.1-0.20190924171022-f4c36041199d/go.mod h1:T56HUNYZUQ1AGUzhAYPugZfp36sKApVnGBgKlIY+aIE= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= +golang.org/x/crypto v0.0.0-20180527072434-ab813273cd59 h1:hk3yo72LXLapY9EXVttc3Z1rLOxT9IuAPPX3GpY2+jo= +golang.org/x/crypto v0.0.0-20180527072434-ab813273cd59/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sys v0.0.0-20180525142821-c11f84a56e43 h1:PvnWIWTbA7gsEBkKjt0HV9hckYfcqYv8s/ju7ArZ0do= +golang.org/x/sys v0.0.0-20180525142821-c11f84a56e43/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +willnorris.com/go/webmention v0.0.0-20200126231626-5a55fff6bf71 h1:F//bgirx4BIsJYHrqAYCZHODn0gSRej/KfueFlarXhs= +willnorris.com/go/webmention v0.0.0-20200126231626-5a55fff6bf71/go.mod h1:p+ZRAsZS2pzZ6kX3GKWYurf3WZI2ygj7VbR8NM8qwfM= diff --git a/http.go b/http.go new file mode 100644 index 0000000..ea1aa25 --- /dev/null +++ b/http.go @@ -0,0 +1,93 @@ +package main + +import ( + "encoding/json" + "github.com/gorilla/mux" + "io/ioutil" + "log" + "net/http" + "net/url" + "regexp" + "willnorris.com/go/webmention" +) + +func Serve() { + + webfingerHandler := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/jrd+json; charset=utf-8") + name := r.URL.Query().Get("resource") // should be something like acct:user@example.com + urlBaseUrl, _ := url.Parse(baseURL) + re := regexp.MustCompile(`^acct:(.*)@` + regexp.QuoteMeta(urlBaseUrl.Host) + `$`) + name = re.ReplaceAllString(name, "$1") + actor := actors[name] + // error out if this actor does not exist + if actor == nil { + w.WriteHeader(http.StatusNotFound) + return + } + responseMap := make(map[string]interface{}) + responseMap["subject"] = "acct:" + actor.Name + "@" + urlBaseUrl.Host + // links is a json array with a single element + var links [1]map[string]string + link1 := make(map[string]string) + link1["rel"] = "self" + link1["type"] = ContentTypeAs2 + link1["href"] = actor.iri + links[0] = link1 + responseMap["links"] = links + response, _ := json.Marshal(responseMap) + _, _ = w.Write(response) + } + + inboxHandler := func(w http.ResponseWriter, r *http.Request) { + b, err := ioutil.ReadAll(r.Body) + if err != nil { + panic(err) + } + activity := make(map[string]interface{}) + err = json.Unmarshal(b, &activity) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + if activity["type"] == "Follow" { + actor := actors[mux.Vars(r)["actor"]] + // error out if this actor does not exist + if actor == nil { + w.WriteHeader(http.StatusNotFound) + return + } + actor.Accept(activity) + } else if activity["type"] == "Undo" { + // TODO: Implement unfollow + } else if activity["type"] == "Create" { + object, ok := activity["object"].(map[string]interface{}) + if ok { + inReplyTo, ok := object["inReplyTo"].(string) + id, ok2 := object["id"].(string) + if ok && ok2 && len(inReplyTo) > 0 && len(id) > 0 { + webmentionClient := webmention.New(&client) + endpoint, err := webmentionClient.DiscoverEndpoint(inReplyTo) + if err != nil || len(endpoint) < 1 { + return + } + _, err = webmentionClient.SendWebmention(endpoint, id, inReplyTo) + if err != nil { + log.Println("Sending webmention to " + inReplyTo + " failed") + return + } + log.Println("Sent webmention to " + inReplyTo) + } + } + } + } + + // Add the handlers to a HTTP server + gorilla := mux.NewRouter() + gorilla.HandleFunc("/.well-known/webfinger", webfingerHandler) + gorilla.HandleFunc("/{actor}/inbox", inboxHandler) + gorilla.HandleFunc("/{actor}/inbox/", inboxHandler) + http.Handle("/", gorilla) + + log.Fatal(http.ListenAndServe(":8081", nil)) +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..0a25493 --- /dev/null +++ b/main.go @@ -0,0 +1,41 @@ +package main + +import ( + "fmt" + "time" +) + +func main() { + Setup() + SetupActors() + go func() { + ticker := time.NewTicker(60 * time.Second) + defer ticker.Stop() + for { + select { + case t := <-ticker.C: + fmt.Println("Fetch feeds: ", t.Format(time.RFC3339)) + for _, actor := range actors { + fmt.Println(actor.feed) + articles, err := allFeedItems(actor.feed) + if err != nil { + fmt.Println(actor.feed, err.Error()) + continue + } + // Prevent map from getting to big + oldSentItems := actor.sentItems + actor.sentItems = make(map[string]bool) + for _, article := range articles { + if oldSentItems[article] == false { + fmt.Println("Send", article) + go actor.PostArticle(article) + } + actor.sentItems[article] = true + } + _ = actor.save() + } + } + } + }() + Serve() +} diff --git a/remoteActor.go b/remoteActor.go new file mode 100644 index 0000000..4b5b91c --- /dev/null +++ b/remoteActor.go @@ -0,0 +1,61 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" +) + +type RemoteActor struct { + iri, inbox, sharedInbox string + info map[string]interface{} +} + +func NewRemoteActor(iri string) (RemoteActor, error) { + info, err := get(iri) + if err != nil { + return RemoteActor{}, err + } + inbox := info["inbox"].(string) + var endpoints map[string]interface{} + var sharedInbox string + if info["endpoints"] != nil { + endpoints = info["endpoints"].(map[string]interface{}) + if val, ok := endpoints["sharedInbox"]; ok { + sharedInbox = val.(string) + } + } + return RemoteActor{ + iri: iri, + inbox: inbox, + sharedInbox: sharedInbox, + }, err +} + +func get(iri string) (info map[string]interface{}, err error) { + buf := new(bytes.Buffer) + req, err := http.NewRequest("GET", iri, buf) + if err != nil { + return + } + req.Header.Add("Accept", ContentTypeAs2) + req.Header.Add("User-Agent", fmt.Sprintf("%s %s", libName, version)) + req.Header.Add("Accept-Charset", "utf-8") + resp, err := client.Do(req) + if err != nil { + return + } + responseData, _ := ioutil.ReadAll(resp.Body) + if !isSuccess(resp.StatusCode) { + return + } + var e interface{} + err = json.Unmarshal(responseData, &e) + if err != nil { + return + } + info = e.(map[string]interface{}) + return +} diff --git a/setup.go b/setup.go new file mode 100644 index 0000000..0bc902a --- /dev/null +++ b/setup.go @@ -0,0 +1,72 @@ +package main + +import ( + "fmt" + "net/http" + "os" + "strings" +) + +var slash = string(os.PathSeparator) +var baseURL string +var storage = "storage" +var actors = make(map[string]*Actor) + +const libName = "jsonpub" +const version = "0.0.1" + +var client = http.Client{} + +// Setup sets our environment up +func Setup() { + // Load base url + baseURL = os.Getenv("BASE_URL") + if len(baseURL) < 1 { + fmt.Printf("BASE_URL not configured") + os.Exit(1) + } + // check if it ends with a / and append one if not + if baseURL[len(baseURL)-1:] != "/" { + baseURL += "/" + } + // print baseURL + fmt.Println("Base URL:", baseURL) + cwd, _ := os.Getwd() + fmt.Println("Storage Location:", cwd+slash+storage) +} + +// Get actors from env vars +// NAMES: names of actors +// {{NAME}}_IRI: IRI of actor +// {{NAME}}_PK: Storage location of private Key of actor +func SetupActors() { + namesString := os.Getenv("NAMES") + if len(namesString) < 1 { + fmt.Printf("NAMES not configured") + os.Exit(1) + } + names := strings.Split(namesString, ",") + for _, name := range names { + iri := os.Getenv(strings.ToUpper(name) + "_IRI") + if len(iri) < 1 { + fmt.Printf(strings.ToUpper(name) + "_IRI not configured") + os.Exit(1) + } + feed := os.Getenv(strings.ToUpper(name) + "_FEED") + if len(feed) < 1 { + fmt.Printf(strings.ToUpper(name) + "_FEED not configured") + os.Exit(1) + } + pk := os.Getenv(strings.ToUpper(name) + "_PK") + if len(pk) < 1 { + fmt.Printf(strings.ToUpper(name) + "_PK not configured") + os.Exit(1) + } + actor, err := GetActor(name, iri, feed, pk) + actors[name] = &actor + if err != nil { + fmt.Printf(err.Error()) + os.Exit(1) + } + } +} \ No newline at end of file diff --git a/util.go b/util.go new file mode 100644 index 0000000..9816ab7 --- /dev/null +++ b/util.go @@ -0,0 +1,18 @@ +package main + +import ( + "net/http" +) + +const ContentTypeAs2 = "application/activity+json" + +func isSuccess(code int) bool { + return code == http.StatusOK || + code == http.StatusCreated || + code == http.StatusAccepted || + code == http.StatusNoContent +} + +func context() [1]string { + return [1]string{"https://www.w3.org/ns/activitystreams"} +}