package main import ( "io" "net/http" "net/url" "sort" "strings" "github.com/PuerkitoBio/goquery" "github.com/araddon/dateparse" "github.com/thoas/go-funk" ) type requestContextKey string func urlize(str string) string { var sb strings.Builder for _, c := range strings.ToLower(str) { if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' { _, _ = sb.WriteRune(c) } else if c == ' ' { _, _ = sb.WriteRune('-') } } return sb.String() } func sortedStrings(s []string) []string { sort.Slice(s, func(i, j int) bool { return strings.ToLower(s[i]) < strings.ToLower(s[j]) }) return s } const randomLetters = "abcdefghijklmnopqrstuvwxyz" func generateRandomString(chars int) string { return funk.RandomString(chars, []rune(randomLetters)) } func isAllowedHost(r *http.Request, hosts ...string) bool { if r.URL == nil { return false } rh := r.URL.Host switch r.URL.Scheme { case "http": rh = strings.TrimSuffix(rh, ":80") case "https": rh = strings.TrimSuffix(rh, ":443") } for _, host := range hosts { if rh == host { return true } } return false } func isAbsoluteURL(s string) bool { if !strings.HasPrefix(s, "https://") && !strings.HasPrefix(s, "http://") { return false } if _, err := url.Parse(s); err != nil { return false } return true } func allLinksFromHTML(r io.Reader, baseURL string) ([]string, error) { doc, err := goquery.NewDocumentFromReader(r) if err != nil { return nil, err } links := []string{} doc.Find("a[href]").Each(func(_ int, item *goquery.Selection) { if href, exists := item.Attr("href"); exists { links = append(links, href) } }) links, err = resolveURLReferences(baseURL, links...) return funk.UniqString(links), err } func resolveURLReferences(base string, refs ...string) ([]string, error) { b, err := url.Parse(base) if err != nil { return nil, err } var urls []string for _, r := range refs { u, err := url.Parse(r) if err != nil { continue } urls = append(urls, b.ResolveReference(u).String()) } return urls, nil } func unescapedPath(p string) string { if u, err := url.PathUnescape(p); err == nil { return u } return p } func slashIfEmpty(s string) string { if s == "" { return "/" } return s } type stringGroup struct { Identifier string Strings []string } func groupStrings(toGroup []string) []stringGroup { stringMap := map[string][]string{} for _, s := range toGroup { first := strings.ToUpper(string([]rune(s)[0])) stringMap[first] = append(stringMap[first], s) } stringGroups := []stringGroup{} for key, sa := range stringMap { stringGroups = append(stringGroups, stringGroup{ Identifier: key, Strings: sortedStrings(sa), }) } sort.Slice(stringGroups, func(i, j int) bool { return strings.ToLower(stringGroups[i].Identifier) < strings.ToLower(stringGroups[j].Identifier) }) return stringGroups } func toLocalSafe(s string) string { d, _ := toLocal(s) return d } func toLocal(s string) (string, error) { if s == "" { return "", nil } d, err := dateparse.ParseLocal(s) if err != nil { return "", err } return d.Local().String(), nil } func dateFormat(date string, format string) string { d, err := dateparse.ParseLocal(date) if err != nil { return "" } return d.Local().Format(format) } type stringPair struct { First, Second string } func wordCount(s string) int { return len(strings.Fields(s)) }