2020-08-05 17:14:10 +00:00
|
|
|
package main
|
|
|
|
|
2020-09-01 16:53:21 +00:00
|
|
|
import (
|
2021-07-12 14:19:28 +00:00
|
|
|
"bytes"
|
2021-06-20 13:18:02 +00:00
|
|
|
"crypto/sha256"
|
|
|
|
"fmt"
|
2021-06-18 12:32:03 +00:00
|
|
|
"html/template"
|
2020-11-16 13:18:14 +00:00
|
|
|
"io"
|
|
|
|
"net/url"
|
2021-06-20 13:18:02 +00:00
|
|
|
"path"
|
2020-09-19 12:56:31 +00:00
|
|
|
"sort"
|
2020-09-01 16:53:21 +00:00
|
|
|
"strings"
|
2021-06-18 12:32:03 +00:00
|
|
|
"time"
|
2021-06-15 15:36:41 +00:00
|
|
|
"unicode"
|
2020-11-16 13:18:14 +00:00
|
|
|
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
2020-12-16 20:24:53 +00:00
|
|
|
"github.com/araddon/dateparse"
|
2021-06-28 20:17:18 +00:00
|
|
|
"github.com/c2h5oh/datasize"
|
2021-04-16 18:00:38 +00:00
|
|
|
"github.com/thoas/go-funk"
|
2020-09-01 16:53:21 +00:00
|
|
|
)
|
|
|
|
|
2021-06-29 15:07:08 +00:00
|
|
|
type contextKey string
|
2021-02-08 17:51:07 +00:00
|
|
|
|
2020-09-01 16:53:21 +00:00
|
|
|
func urlize(str string) string {
|
2021-02-24 12:16:33 +00:00
|
|
|
var sb strings.Builder
|
|
|
|
for _, c := range strings.ToLower(str) {
|
|
|
|
if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' {
|
|
|
|
_, _ = sb.WriteRune(c)
|
|
|
|
} else if c == ' ' {
|
|
|
|
_, _ = sb.WriteRune('-')
|
2020-09-01 16:53:21 +00:00
|
|
|
}
|
|
|
|
}
|
2021-02-24 12:16:33 +00:00
|
|
|
return sb.String()
|
2020-09-01 16:53:21 +00:00
|
|
|
}
|
2020-09-15 15:01:53 +00:00
|
|
|
|
2020-09-19 12:56:31 +00:00
|
|
|
func sortedStrings(s []string) []string {
|
|
|
|
sort.Slice(s, func(i, j int) bool {
|
|
|
|
return strings.ToLower(s[i]) < strings.ToLower(s[j])
|
|
|
|
})
|
|
|
|
return s
|
|
|
|
}
|
2020-10-06 17:07:48 +00:00
|
|
|
|
2021-03-11 08:16:19 +00:00
|
|
|
const randomLetters = "abcdefghijklmnopqrstuvwxyz"
|
|
|
|
|
2020-10-26 16:37:31 +00:00
|
|
|
func generateRandomString(chars int) string {
|
2021-04-16 18:00:38 +00:00
|
|
|
return funk.RandomString(chars, []rune(randomLetters))
|
2020-10-06 17:07:48 +00:00
|
|
|
}
|
2020-11-01 17:37:21 +00:00
|
|
|
|
2020-11-16 13:18:14 +00:00
|
|
|
func isAbsoluteURL(s string) bool {
|
2021-06-14 14:29:22 +00:00
|
|
|
if u, err := url.Parse(s); err != nil || !u.IsAbs() {
|
2020-11-16 13:18:14 +00:00
|
|
|
return false
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
2021-06-14 14:29:22 +00:00
|
|
|
func allLinksFromHTMLString(html, baseURL string) ([]string, error) {
|
|
|
|
return allLinksFromHTML(strings.NewReader(html), baseURL)
|
|
|
|
}
|
|
|
|
|
2020-11-16 13:18:14 +00:00
|
|
|
func allLinksFromHTML(r io.Reader, baseURL string) ([]string, error) {
|
|
|
|
doc, err := goquery.NewDocumentFromReader(r)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
links := []string{}
|
|
|
|
doc.Find("a[href]").Each(func(_ int, item *goquery.Selection) {
|
|
|
|
if href, exists := item.Attr("href"); exists {
|
2020-11-16 17:34:29 +00:00
|
|
|
links = append(links, href)
|
2020-11-16 13:18:14 +00:00
|
|
|
}
|
|
|
|
})
|
2020-11-16 17:34:29 +00:00
|
|
|
links, err = resolveURLReferences(baseURL, links...)
|
2021-04-16 18:00:38 +00:00
|
|
|
return funk.UniqString(links), err
|
2020-11-16 17:34:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func resolveURLReferences(base string, refs ...string) ([]string, error) {
|
|
|
|
b, err := url.Parse(base)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
var urls []string
|
|
|
|
for _, r := range refs {
|
|
|
|
u, err := url.Parse(r)
|
|
|
|
if err != nil {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
urls = append(urls, b.ResolveReference(u).String())
|
|
|
|
}
|
|
|
|
return urls, nil
|
2020-11-16 13:18:14 +00:00
|
|
|
}
|
2020-11-17 16:43:30 +00:00
|
|
|
|
2020-11-17 19:01:02 +00:00
|
|
|
func unescapedPath(p string) string {
|
2021-02-24 12:16:33 +00:00
|
|
|
if u, err := url.PathUnescape(p); err == nil {
|
|
|
|
return u
|
2020-11-17 19:01:02 +00:00
|
|
|
}
|
2021-02-24 12:16:33 +00:00
|
|
|
return p
|
2020-11-17 19:01:02 +00:00
|
|
|
}
|
|
|
|
|
2020-11-22 08:11:57 +00:00
|
|
|
type stringGroup struct {
|
|
|
|
Identifier string
|
|
|
|
Strings []string
|
|
|
|
}
|
|
|
|
|
|
|
|
func groupStrings(toGroup []string) []stringGroup {
|
|
|
|
stringMap := map[string][]string{}
|
|
|
|
for _, s := range toGroup {
|
2021-03-26 08:33:46 +00:00
|
|
|
first := strings.ToUpper(string([]rune(s)[0]))
|
2020-11-22 08:11:57 +00:00
|
|
|
stringMap[first] = append(stringMap[first], s)
|
|
|
|
}
|
|
|
|
stringGroups := []stringGroup{}
|
|
|
|
for key, sa := range stringMap {
|
|
|
|
stringGroups = append(stringGroups, stringGroup{
|
|
|
|
Identifier: key,
|
|
|
|
Strings: sortedStrings(sa),
|
|
|
|
})
|
|
|
|
}
|
|
|
|
sort.Slice(stringGroups, func(i, j int) bool {
|
|
|
|
return strings.ToLower(stringGroups[i].Identifier) < strings.ToLower(stringGroups[j].Identifier)
|
|
|
|
})
|
|
|
|
return stringGroups
|
|
|
|
}
|
2020-12-16 20:24:53 +00:00
|
|
|
|
|
|
|
func toLocalSafe(s string) string {
|
|
|
|
d, _ := toLocal(s)
|
|
|
|
return d
|
|
|
|
}
|
|
|
|
|
|
|
|
func toLocal(s string) (string, error) {
|
|
|
|
if s == "" {
|
|
|
|
return "", nil
|
|
|
|
}
|
|
|
|
d, err := dateparse.ParseLocal(s)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
2021-07-13 15:23:10 +00:00
|
|
|
return d.Local().Format(time.RFC3339), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func toUTCSafe(s string) string {
|
|
|
|
d, _ := toUTC(s)
|
|
|
|
return d
|
|
|
|
}
|
|
|
|
|
|
|
|
func toUTC(s string) (string, error) {
|
|
|
|
if s == "" {
|
|
|
|
return "", nil
|
|
|
|
}
|
|
|
|
d, err := dateparse.ParseLocal(s)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return d.UTC().Format(time.RFC3339), nil
|
2020-12-16 20:24:53 +00:00
|
|
|
}
|
2021-02-26 09:31:38 +00:00
|
|
|
|
|
|
|
func dateFormat(date string, format string) string {
|
|
|
|
d, err := dateparse.ParseLocal(date)
|
|
|
|
if err != nil {
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
return d.Local().Format(format)
|
|
|
|
}
|
2021-04-03 13:39:43 +00:00
|
|
|
|
2021-06-18 12:32:03 +00:00
|
|
|
func isoDateFormat(date string) string {
|
|
|
|
return dateFormat(date, "2006-01-02")
|
|
|
|
}
|
|
|
|
|
|
|
|
func unixToLocalDateString(unix int64) string {
|
2021-07-13 15:23:10 +00:00
|
|
|
return time.Unix(unix, 0).Local().Format(time.RFC3339)
|
2021-06-18 12:32:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func localNowString() string {
|
2021-07-13 15:23:10 +00:00
|
|
|
return time.Now().Local().Format(time.RFC3339)
|
|
|
|
}
|
|
|
|
|
|
|
|
func utcNowString() string {
|
|
|
|
return time.Now().UTC().Format(time.RFC3339)
|
2021-06-18 12:32:03 +00:00
|
|
|
}
|
|
|
|
|
2021-04-03 13:39:43 +00:00
|
|
|
type stringPair struct {
|
2021-04-23 18:52:12 +00:00
|
|
|
First, Second string
|
2021-04-03 13:39:43 +00:00
|
|
|
}
|
2021-04-28 18:03:20 +00:00
|
|
|
|
|
|
|
func wordCount(s string) int {
|
|
|
|
return len(strings.Fields(s))
|
|
|
|
}
|
2021-06-15 15:36:41 +00:00
|
|
|
|
|
|
|
// Count all letters and numbers in string
|
|
|
|
func charCount(s string) (count int) {
|
|
|
|
for _, r := range s {
|
|
|
|
if unicode.IsLetter(r) || unicode.IsNumber(r) {
|
|
|
|
count++
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return count
|
|
|
|
}
|
2021-06-18 12:32:03 +00:00
|
|
|
|
|
|
|
func wrapStringAsHTML(s string) template.HTML {
|
|
|
|
return template.HTML(s)
|
|
|
|
}
|
2021-06-20 13:18:02 +00:00
|
|
|
|
|
|
|
// Check if url has allowed file extension
|
|
|
|
func urlHasExt(rawUrl string, allowed ...string) (ext string, has bool) {
|
|
|
|
u, err := url.Parse(rawUrl)
|
|
|
|
if err != nil {
|
|
|
|
return "", false
|
|
|
|
}
|
|
|
|
ext = strings.ToLower(path.Ext(u.Path))
|
|
|
|
if ext == "" {
|
|
|
|
return "", false
|
|
|
|
}
|
|
|
|
ext = ext[1:]
|
|
|
|
allowed = funk.Map(allowed, func(str string) string {
|
|
|
|
return strings.ToLower(str)
|
|
|
|
}).([]string)
|
|
|
|
return ext, funk.ContainsString(allowed, strings.ToLower(ext))
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get SHA-256 hash of file
|
|
|
|
func getSHA256(file io.ReadSeeker) (filename string, err error) {
|
|
|
|
if _, err = file.Seek(0, 0); err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
h := sha256.New()
|
|
|
|
if _, err = io.Copy(h, file); err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
if _, err = file.Seek(0, 0); err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return fmt.Sprintf("%x", h.Sum(nil)), nil
|
|
|
|
}
|
2021-06-28 20:17:18 +00:00
|
|
|
|
|
|
|
func mBytesString(size int64) string {
|
|
|
|
return fmt.Sprintf("%.2f MB", datasize.ByteSize(size).MBytes())
|
|
|
|
}
|
2021-07-12 14:19:28 +00:00
|
|
|
|
|
|
|
func htmlText(b []byte) string {
|
|
|
|
d, err := goquery.NewDocumentFromReader(bytes.NewReader(b))
|
|
|
|
if err != nil {
|
|
|
|
return ""
|
|
|
|
}
|
|
|
|
return strings.TrimSpace(d.Text())
|
|
|
|
}
|
|
|
|
|
|
|
|
func defaultIfEmpty(s, d string) string {
|
|
|
|
if s != "" {
|
|
|
|
return s
|
|
|
|
}
|
|
|
|
return d
|
|
|
|
}
|
2021-07-23 15:26:14 +00:00
|
|
|
|
|
|
|
func containsStrings(s string, subStrings ...string) bool {
|
|
|
|
for _, ss := range subStrings {
|
|
|
|
if strings.Contains(s, ss) {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
2021-07-27 10:51:08 +00:00
|
|
|
|
|
|
|
func timeNoErr(t time.Time, _ error) time.Time {
|
|
|
|
return t
|
|
|
|
}
|