You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
158 lines
3.1 KiB
158 lines
3.1 KiB
package main
|
|
|
|
import (
|
|
"io"
|
|
"math/rand"
|
|
"net/http"
|
|
"net/url"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
"github.com/araddon/dateparse"
|
|
)
|
|
|
|
func urlize(str string) string {
|
|
newStr := ""
|
|
for _, c := range strings.Split(strings.ToLower(str), "") {
|
|
if c >= "a" && c <= "z" || c >= "A" && c <= "Z" || c >= "0" && c <= "9" {
|
|
newStr += c
|
|
} else if c == " " {
|
|
newStr += "-"
|
|
}
|
|
}
|
|
return newStr
|
|
}
|
|
|
|
func sortedStrings(s []string) []string {
|
|
sort.Slice(s, func(i, j int) bool {
|
|
return strings.ToLower(s[i]) < strings.ToLower(s[j])
|
|
})
|
|
return s
|
|
}
|
|
|
|
func generateRandomString(chars int) string {
|
|
rand.Seed(time.Now().UnixNano())
|
|
letters := []rune("abcdefghijklmnopqrstuvwxyz")
|
|
b := make([]rune, chars)
|
|
for i := range b {
|
|
b[i] = letters[rand.Intn(len(letters))]
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
func isAllowedHost(r *http.Request, hosts ...string) bool {
|
|
if r.URL == nil {
|
|
return false
|
|
}
|
|
rh := r.URL.Host
|
|
switch r.URL.Scheme {
|
|
case "http":
|
|
rh = strings.TrimSuffix(rh, ":80")
|
|
case "https":
|
|
rh = strings.TrimSuffix(rh, ":443")
|
|
}
|
|
for _, host := range hosts {
|
|
if rh == host {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func isAbsoluteURL(s string) bool {
|
|
if !strings.HasPrefix(s, "https://") && !strings.HasPrefix(s, "http://") {
|
|
return false
|
|
}
|
|
if _, err := url.Parse(s); err != nil {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func allLinksFromHTML(r io.Reader, baseURL string) ([]string, error) {
|
|
doc, err := goquery.NewDocumentFromReader(r)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
links := []string{}
|
|
doc.Find("a[href]").Each(func(_ int, item *goquery.Selection) {
|
|
if href, exists := item.Attr("href"); exists {
|
|
links = append(links, href)
|
|
}
|
|
})
|
|
links, err = resolveURLReferences(baseURL, links...)
|
|
return links, err
|
|
}
|
|
|
|
func resolveURLReferences(base string, refs ...string) ([]string, error) {
|
|
b, err := url.Parse(base)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var urls []string
|
|
for _, r := range refs {
|
|
u, err := url.Parse(r)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
urls = append(urls, b.ResolveReference(u).String())
|
|
}
|
|
return urls, nil
|
|
}
|
|
|
|
func unescapedPath(p string) string {
|
|
u, err := url.PathUnescape(p)
|
|
if err != nil {
|
|
return p
|
|
}
|
|
return u
|
|
}
|
|
|
|
func slashIfEmpty(s string) string {
|
|
if s == "" {
|
|
return "/"
|
|
}
|
|
return s
|
|
}
|
|
|
|
type stringGroup struct {
|
|
Identifier string
|
|
Strings []string
|
|
}
|
|
|
|
func groupStrings(toGroup []string) []stringGroup {
|
|
stringMap := map[string][]string{}
|
|
for _, s := range toGroup {
|
|
first := strings.ToUpper(strings.Split(s, "")[0])
|
|
stringMap[first] = append(stringMap[first], s)
|
|
}
|
|
stringGroups := []stringGroup{}
|
|
for key, sa := range stringMap {
|
|
stringGroups = append(stringGroups, stringGroup{
|
|
Identifier: key,
|
|
Strings: sortedStrings(sa),
|
|
})
|
|
}
|
|
sort.Slice(stringGroups, func(i, j int) bool {
|
|
return strings.ToLower(stringGroups[i].Identifier) < strings.ToLower(stringGroups[j].Identifier)
|
|
})
|
|
return stringGroups
|
|
}
|
|
|
|
func toLocalSafe(s string) string {
|
|
d, _ := toLocal(s)
|
|
return d
|
|
}
|
|
|
|
func toLocal(s string) (string, error) {
|
|
if s == "" {
|
|
return "", nil
|
|
}
|
|
d, err := dateparse.ParseLocal(s)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return d.Local().String(), nil
|
|
}
|