commit 23f8e8aaad678d107f71dc8d6788f25b4651c462 Author: Jan-Lukas Else Date: Tue Feb 4 21:25:57 2020 +0100 Init diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a2a9842 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +storage +jsonpub +.idea +private.pem +public.pem \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1c944ae --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.13-alpine as build +ADD . /app +WORKDIR /app +RUN go build + +FROM alpine:3.11 +RUN apk add --no-cache tzdata ca-certificates +COPY --from=build /app/jsonpub /bin/ +WORKDIR /app +VOLUME /app/storage +EXPOSE 8081 +CMD ["jsonpub"] \ No newline at end of file diff --git a/actor.go b/actor.go new file mode 100644 index 0000000..2f84fc9 --- /dev/null +++ b/actor.go @@ -0,0 +1,250 @@ +package main + +import ( + "bytes" + "crypto" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "github.com/dchest/uniuri" + "github.com/go-fed/httpsig" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +type Actor struct { + Name, iri, feed string + followers map[string]interface{} + sentItems map[string]bool + privateKey crypto.PrivateKey + publicKeyID string +} + +type ActorToSave struct { + Name string + Followers map[string]interface{} + SentItems map[string]bool +} + +func GetActor(name, iri, feed, pk string) (Actor, error) { + // make sure users can't read hard drive + if strings.ContainsAny(name, "./ ") { + return Actor{}, errors.New("illegal characters in actor name") + } + // populate actor + pkfile, err := ioutil.ReadFile(pk) + if err != nil { + return Actor{}, errors.New("failed to read private key file") + } + privateKeyDecoded, rest := pem.Decode(pkfile) + if privateKeyDecoded == nil { + fmt.Println(rest) + return Actor{}, errors.New("failed to decode the private key PEM") + } + privateKey, err := x509.ParsePKCS1PrivateKey(privateKeyDecoded.Bytes) + if err != nil { + return Actor{}, errors.New("failed to parse the private key") + } + actor := Actor{ + Name: name, + iri: iri, + feed: feed, + followers: make(map[string]interface{}), + sentItems: make(map[string]bool), + privateKey: privateKey, + publicKeyID: iri + "#main-key", + } + jsonFile := storage + slash + "actors" + slash + name + slash + name + ".json" + fileHandle, err := os.Open(jsonFile) + if os.IsNotExist(err) { + // File doesn't exist, maybe it's a new actor + return actor, nil + } + byteValue, err := ioutil.ReadAll(fileHandle) + if err != nil { + // Ignore error, but return actor + return actor, nil + } + savedActor := ActorToSave{} + err = json.Unmarshal(byteValue, &savedActor) + if err != nil { + // Ignore error, but return actor + return actor, nil + } + if savedActor.Followers != nil { + actor.followers = savedActor.Followers + } + if savedActor.SentItems != nil { + actor.sentItems = savedActor.SentItems + } + return actor, nil +} + +// save the actor to file +func (a *Actor) save() error { + // check if directory already exists + dir := storage + slash + "actors" + slash + a.Name + if _, err := os.Stat(dir); os.IsNotExist(err) { + _ = os.MkdirAll(dir, 0755) + } + actorToSave := ActorToSave{ + Name: a.Name, + Followers: a.followers, + SentItems: a.sentItems, + } + actorJSON, err := json.MarshalIndent(actorToSave, "", "\t") + if err != nil { + return err + } + err = ioutil.WriteFile(storage+slash+"actors"+slash+a.Name+slash+a.Name+".json", actorJSON, 0644) + if err != nil { + return err + } + return nil +} + +func (a *Actor) PostArticle(url string) { + create := make(map[string]interface{}) + note := make(map[string]interface{}) + create["@context"] = context() + create["actor"] = a.iri + create["id"] = url + create["published"] = note["published"] + create["type"] = "Create" + article := make(map[string]interface{}) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return + } + req.Header.Add("User-Agent", fmt.Sprintf("%s %s", libName, version)) + req.Header.Add("Accept", ContentTypeAs2) + resp, err := client.Do(req) + if err != nil || isSuccess(resp.StatusCode) { + fmt.Println("Failed to fetch article") + return + } + respBody, err := ioutil.ReadAll(resp.Body) + defer func() { _ = resp.Body.Close() }() + if err != nil { + fmt.Println("Failed to read response from fetched article") + return + } + err = json.Unmarshal(respBody, &article) + if err != nil { + fmt.Println("Failed to unmarshal fetched article") + return + } + create["object"] = article + go func() { _ = a.sendToFollowers(create) }() +} + +// signedHTTPPost performs an HTTP post on behalf of Actor with the +// request-target, date, host and digest headers signed +// with the actor's private key. +func (a *Actor) signedHTTPPost(content map[string]interface{}, to string) (err error) { + b, err := json.Marshal(content) + if err != nil { + return + } + postSigner, _, _ := httpsig.NewSigner([]httpsig.Algorithm{httpsig.RSA_SHA256}, "SHA-256", []string{"(request-target)", "date", "host", "digest"}, httpsig.Signature) + byteCopy := make([]byte, len(b)) + copy(byteCopy, b) + buf := bytes.NewBuffer(byteCopy) + req, err := http.NewRequest(http.MethodPost, to, buf) + if err != nil { + return + } + iri, err := url.Parse(to) + if err != nil { + return err + } + req.Header.Add("Accept-Charset", "utf-8") + req.Header.Add("Date", time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT") + req.Header.Add("User-Agent", fmt.Sprintf("%s %s", libName, version)) + req.Header.Add("Host", iri.Host) + req.Header.Add("Accept", "application/activity+json; charset=utf-8") + req.Header.Add("Content-Type", "application/activity+json; charset=utf-8") + err = postSigner.SignRequest(a.privateKey, a.publicKeyID, req, byteCopy) + if err != nil { + return + } + resp, err := client.Do(req) + if err != nil { + return + } + if !isSuccess(resp.StatusCode) { + err = errors.New("post request failed") + return + } + return +} + +// NewFollower records a new follower to the actor file +func (a *Actor) NewFollower(iri string, inbox string) error { + a.followers[iri] = inbox + return a.save() +} + +func (a *Actor) RemoveFollower(iri string) error { + delete(a.followers, iri) + return a.save() +} + +// batchSend sends a batch of http posts to a list of recipients +func (a *Actor) batchSend(activity map[string]interface{}, recipients []string) (err error) { + for _, v := range recipients { + _ = a.signedHTTPPost(activity, v) + } + return +} + +// send to followers sends a batch of http posts to each one of the followers +func (a *Actor) sendToFollowers(activity map[string]interface{}) (err error) { + recipients := make([]string, len(a.followers)) + i := 0 + for _, inbox := range a.followers { + recipients[i] = inbox.(string) + i++ + } + _ = a.batchSend(activity, recipients) + return +} + +func (a *Actor) newID() (hash string, url string) { + hash = uniuri.New() + return hash, a.iri + hash +} + +// Accept a follow request +func (a *Actor) Accept(follow map[string]interface{}) { + // it's a follow, write it down + newFollower := follow["actor"].(string) + // check we aren't following ourselves + if newFollower == follow["object"] { + // actor and object are equal + return + } + follower, err := NewRemoteActor(follow["actor"].(string)) + if err != nil { + // Couldn't retrieve remote actor info + return + } + // Add or update follower + _ = a.NewFollower(newFollower, follower.inbox) + // remove @context from the inner activity + delete(follow, "@context") + accept := make(map[string]interface{}) + accept["@context"] = "https://www.w3.org/ns/activitystreams" + accept["to"] = follow["actor"] + _, accept["id"] = a.newID() + accept["actor"] = a.iri + accept["object"] = follow + accept["type"] = "Accept" + go func() { _ = a.signedHTTPPost(accept, follower.inbox) }() +} diff --git a/config.ini b/config.ini new file mode 100644 index 0000000..bfbbe2b --- /dev/null +++ b/config.ini @@ -0,0 +1,2 @@ +[general] +baseURL = https://jlelse.blog \ No newline at end of file diff --git a/feed.go b/feed.go new file mode 100644 index 0000000..49865b7 --- /dev/null +++ b/feed.go @@ -0,0 +1,33 @@ +package main + +import ( + "encoding/json" + "errors" + "io/ioutil" +) + +func allFeedItems(url string) ([]string, error) { + jsonFeed := &struct { + Items []struct { + Url string `json:"url"` + } `json:"items"` + }{} + resp, err := client.Get(url) + if err != nil { + return nil, errors.New("failed to get json feed") + } + defer func() { _ = resp.Body.Close() }() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.New("failed to read json feed") + } + err = json.Unmarshal(body, &jsonFeed) + if err != nil { + return nil, errors.New("failed to parse json feed") + } + var allUrls []string + for _, item := range jsonFeed.Items { + allUrls = append(allUrls, item.Url) + } + return allUrls, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6fcfd8f --- /dev/null +++ b/go.mod @@ -0,0 +1,11 @@ +module codeberg.org/jlelse/jsonpub + +go 1.13 + +require ( + github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 + github.com/go-fed/httpsig v0.1.1-0.20190924171022-f4c36041199d + github.com/gorilla/mux v1.7.3 + golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 // indirect + willnorris.com/go/webmention v0.0.0-20200126231626-5a55fff6bf71 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..b3f75f6 --- /dev/null +++ b/go.sum @@ -0,0 +1,24 @@ +github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 h1:74lLNRzvsdIlkTgfDSMuaPjBr4cf6k7pwQQANm/yLKU= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= +github.com/go-fed/httpsig v0.1.1-0.20190924171022-f4c36041199d h1:+uoOvOnNDgsYbWtAij4xP6Rgir3eJGjocFPxBJETU/U= +github.com/go-fed/httpsig v0.1.1-0.20190924171022-f4c36041199d/go.mod h1:T56HUNYZUQ1AGUzhAYPugZfp36sKApVnGBgKlIY+aIE= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= +golang.org/x/crypto v0.0.0-20180527072434-ab813273cd59 h1:hk3yo72LXLapY9EXVttc3Z1rLOxT9IuAPPX3GpY2+jo= +golang.org/x/crypto v0.0.0-20180527072434-ab813273cd59/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sys v0.0.0-20180525142821-c11f84a56e43 h1:PvnWIWTbA7gsEBkKjt0HV9hckYfcqYv8s/ju7ArZ0do= +golang.org/x/sys v0.0.0-20180525142821-c11f84a56e43/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +willnorris.com/go/webmention v0.0.0-20200126231626-5a55fff6bf71 h1:F//bgirx4BIsJYHrqAYCZHODn0gSRej/KfueFlarXhs= +willnorris.com/go/webmention v0.0.0-20200126231626-5a55fff6bf71/go.mod h1:p+ZRAsZS2pzZ6kX3GKWYurf3WZI2ygj7VbR8NM8qwfM= diff --git a/http.go b/http.go new file mode 100644 index 0000000..ea1aa25 --- /dev/null +++ b/http.go @@ -0,0 +1,93 @@ +package main + +import ( + "encoding/json" + "github.com/gorilla/mux" + "io/ioutil" + "log" + "net/http" + "net/url" + "regexp" + "willnorris.com/go/webmention" +) + +func Serve() { + + webfingerHandler := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/jrd+json; charset=utf-8") + name := r.URL.Query().Get("resource") // should be something like acct:user@example.com + urlBaseUrl, _ := url.Parse(baseURL) + re := regexp.MustCompile(`^acct:(.*)@` + regexp.QuoteMeta(urlBaseUrl.Host) + `$`) + name = re.ReplaceAllString(name, "$1") + actor := actors[name] + // error out if this actor does not exist + if actor == nil { + w.WriteHeader(http.StatusNotFound) + return + } + responseMap := make(map[string]interface{}) + responseMap["subject"] = "acct:" + actor.Name + "@" + urlBaseUrl.Host + // links is a json array with a single element + var links [1]map[string]string + link1 := make(map[string]string) + link1["rel"] = "self" + link1["type"] = ContentTypeAs2 + link1["href"] = actor.iri + links[0] = link1 + responseMap["links"] = links + response, _ := json.Marshal(responseMap) + _, _ = w.Write(response) + } + + inboxHandler := func(w http.ResponseWriter, r *http.Request) { + b, err := ioutil.ReadAll(r.Body) + if err != nil { + panic(err) + } + activity := make(map[string]interface{}) + err = json.Unmarshal(b, &activity) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + if activity["type"] == "Follow" { + actor := actors[mux.Vars(r)["actor"]] + // error out if this actor does not exist + if actor == nil { + w.WriteHeader(http.StatusNotFound) + return + } + actor.Accept(activity) + } else if activity["type"] == "Undo" { + // TODO: Implement unfollow + } else if activity["type"] == "Create" { + object, ok := activity["object"].(map[string]interface{}) + if ok { + inReplyTo, ok := object["inReplyTo"].(string) + id, ok2 := object["id"].(string) + if ok && ok2 && len(inReplyTo) > 0 && len(id) > 0 { + webmentionClient := webmention.New(&client) + endpoint, err := webmentionClient.DiscoverEndpoint(inReplyTo) + if err != nil || len(endpoint) < 1 { + return + } + _, err = webmentionClient.SendWebmention(endpoint, id, inReplyTo) + if err != nil { + log.Println("Sending webmention to " + inReplyTo + " failed") + return + } + log.Println("Sent webmention to " + inReplyTo) + } + } + } + } + + // Add the handlers to a HTTP server + gorilla := mux.NewRouter() + gorilla.HandleFunc("/.well-known/webfinger", webfingerHandler) + gorilla.HandleFunc("/{actor}/inbox", inboxHandler) + gorilla.HandleFunc("/{actor}/inbox/", inboxHandler) + http.Handle("/", gorilla) + + log.Fatal(http.ListenAndServe(":8081", nil)) +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..0a25493 --- /dev/null +++ b/main.go @@ -0,0 +1,41 @@ +package main + +import ( + "fmt" + "time" +) + +func main() { + Setup() + SetupActors() + go func() { + ticker := time.NewTicker(60 * time.Second) + defer ticker.Stop() + for { + select { + case t := <-ticker.C: + fmt.Println("Fetch feeds: ", t.Format(time.RFC3339)) + for _, actor := range actors { + fmt.Println(actor.feed) + articles, err := allFeedItems(actor.feed) + if err != nil { + fmt.Println(actor.feed, err.Error()) + continue + } + // Prevent map from getting to big + oldSentItems := actor.sentItems + actor.sentItems = make(map[string]bool) + for _, article := range articles { + if oldSentItems[article] == false { + fmt.Println("Send", article) + go actor.PostArticle(article) + } + actor.sentItems[article] = true + } + _ = actor.save() + } + } + } + }() + Serve() +} diff --git a/remoteActor.go b/remoteActor.go new file mode 100644 index 0000000..4b5b91c --- /dev/null +++ b/remoteActor.go @@ -0,0 +1,61 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" +) + +type RemoteActor struct { + iri, inbox, sharedInbox string + info map[string]interface{} +} + +func NewRemoteActor(iri string) (RemoteActor, error) { + info, err := get(iri) + if err != nil { + return RemoteActor{}, err + } + inbox := info["inbox"].(string) + var endpoints map[string]interface{} + var sharedInbox string + if info["endpoints"] != nil { + endpoints = info["endpoints"].(map[string]interface{}) + if val, ok := endpoints["sharedInbox"]; ok { + sharedInbox = val.(string) + } + } + return RemoteActor{ + iri: iri, + inbox: inbox, + sharedInbox: sharedInbox, + }, err +} + +func get(iri string) (info map[string]interface{}, err error) { + buf := new(bytes.Buffer) + req, err := http.NewRequest("GET", iri, buf) + if err != nil { + return + } + req.Header.Add("Accept", ContentTypeAs2) + req.Header.Add("User-Agent", fmt.Sprintf("%s %s", libName, version)) + req.Header.Add("Accept-Charset", "utf-8") + resp, err := client.Do(req) + if err != nil { + return + } + responseData, _ := ioutil.ReadAll(resp.Body) + if !isSuccess(resp.StatusCode) { + return + } + var e interface{} + err = json.Unmarshal(responseData, &e) + if err != nil { + return + } + info = e.(map[string]interface{}) + return +} diff --git a/setup.go b/setup.go new file mode 100644 index 0000000..0bc902a --- /dev/null +++ b/setup.go @@ -0,0 +1,72 @@ +package main + +import ( + "fmt" + "net/http" + "os" + "strings" +) + +var slash = string(os.PathSeparator) +var baseURL string +var storage = "storage" +var actors = make(map[string]*Actor) + +const libName = "jsonpub" +const version = "0.0.1" + +var client = http.Client{} + +// Setup sets our environment up +func Setup() { + // Load base url + baseURL = os.Getenv("BASE_URL") + if len(baseURL) < 1 { + fmt.Printf("BASE_URL not configured") + os.Exit(1) + } + // check if it ends with a / and append one if not + if baseURL[len(baseURL)-1:] != "/" { + baseURL += "/" + } + // print baseURL + fmt.Println("Base URL:", baseURL) + cwd, _ := os.Getwd() + fmt.Println("Storage Location:", cwd+slash+storage) +} + +// Get actors from env vars +// NAMES: names of actors +// {{NAME}}_IRI: IRI of actor +// {{NAME}}_PK: Storage location of private Key of actor +func SetupActors() { + namesString := os.Getenv("NAMES") + if len(namesString) < 1 { + fmt.Printf("NAMES not configured") + os.Exit(1) + } + names := strings.Split(namesString, ",") + for _, name := range names { + iri := os.Getenv(strings.ToUpper(name) + "_IRI") + if len(iri) < 1 { + fmt.Printf(strings.ToUpper(name) + "_IRI not configured") + os.Exit(1) + } + feed := os.Getenv(strings.ToUpper(name) + "_FEED") + if len(feed) < 1 { + fmt.Printf(strings.ToUpper(name) + "_FEED not configured") + os.Exit(1) + } + pk := os.Getenv(strings.ToUpper(name) + "_PK") + if len(pk) < 1 { + fmt.Printf(strings.ToUpper(name) + "_PK not configured") + os.Exit(1) + } + actor, err := GetActor(name, iri, feed, pk) + actors[name] = &actor + if err != nil { + fmt.Printf(err.Error()) + os.Exit(1) + } + } +} \ No newline at end of file diff --git a/util.go b/util.go new file mode 100644 index 0000000..9816ab7 --- /dev/null +++ b/util.go @@ -0,0 +1,18 @@ +package main + +import ( + "net/http" +) + +const ContentTypeAs2 = "application/activity+json" + +func isSuccess(code int) bool { + return code == http.StatusOK || + code == http.StatusCreated || + code == http.StatusAccepted || + code == http.StatusNoContent +} + +func context() [1]string { + return [1]string{"https://www.w3.org/ns/activitystreams"} +}