Browse Source

Emojis, subarticles, minification, ...

master
Jan-Lukas Else 7 months ago
parent
commit
c414089951
6 changed files with 181 additions and 20 deletions
  1. +76
    -0
      emoji.go
  2. +2
    -0
      go.mod
  3. +14
    -0
      go.sum
  4. +79
    -18
      main.go
  5. +1
    -1
      markdown.go
  6. +9
    -1
      templates/single.gohtml

+ 76
- 0
emoji.go View File

@ -0,0 +1,76 @@
// This is taken from Hugo
//
// Copyright 2016 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"sync"
"github.com/kyokomi/emoji"
)
var (
emojiInit sync.Once
emojis = make(map[string][]byte)
emojiDelim = []byte(":")
emojiWordDelim = []byte(" ")
emojiMaxSize int
)
func emojify(source []byte) []byte {
emojiInit.Do(initEmoji)
start := 0
k := bytes.Index(source[start:], emojiDelim)
for k != -1 {
j := start + k
upper := j + emojiMaxSize
if upper > len(source) {
upper = len(source)
}
endEmoji := bytes.Index(source[j+1:upper], emojiDelim)
nextWordDelim := bytes.Index(source[j:upper], emojiWordDelim)
if endEmoji < 0 {
start++
} else if endEmoji == 0 || (nextWordDelim != -1 && nextWordDelim < endEmoji) {
start += endEmoji + 1
} else {
endKey := endEmoji + j + 2
emojiKey := source[j:endKey]
if e, ok := emojis[string(emojiKey)]; ok {
source = append(source[:j], append(e, source[endKey:]...)...)
}
start += endEmoji
}
if start >= len(source) {
break
}
k = bytes.Index(source[start:], emojiDelim)
}
return source
}
func initEmoji() {
emojiMap := emoji.CodeMap()
for k, v := range emojiMap {
emojis[k] = []byte(v)
if len(k) > emojiMaxSize {
emojiMaxSize = len(k)
}
}
}

+ 2
- 0
go.mod View File

@ -4,7 +4,9 @@ go 1.14
require (
github.com/kr/text v0.2.0 // indirect
github.com/kyokomi/emoji v2.2.2+incompatible
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/tdewolff/minify/v2 v2.7.4
github.com/yuin/goldmark v1.1.31
github.com/yuin/goldmark-meta v0.0.0-20191126180153-f0638e958b60
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect


+ 14
- 0
go.sum View File

@ -1,16 +1,30 @@
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kyokomi/emoji v2.2.2+incompatible h1:gaQFbK2+uSxOR4iGZprJAbpmtqTrHhSdgOyIMD6Oidc=
github.com/kyokomi/emoji v2.2.2+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/tdewolff/minify/v2 v2.7.4 h1:r0OZQ3QzWeDS5cXq53Bk4IFIBDZ7fiXIkw1a4bHONsw=
github.com/tdewolff/minify/v2 v2.7.4/go.mod h1:BkDSm8aMMT0ALGmpt7j3Ra7nLUgZL0qhyrAHXwxcy5w=
github.com/tdewolff/parse/v2 v2.4.2 h1:Bu2Qv6wepkc+Ou7iB/qHjAhEImlAP5vedzlQRUdj3BI=
github.com/tdewolff/parse/v2 v2.4.2/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho=
github.com/tdewolff/test v1.0.6 h1:76mzYJQ83Op284kMT+63iCNCI7NEERsIN8dLM+RiKr4=
github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
github.com/yuin/goldmark v1.1.7/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.31 h1:nKIhaVknZ0wOBBg0Uu6px+t218SfkLh2i/JwwOXYXqs=
github.com/yuin/goldmark v1.1.31/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark-meta v0.0.0-20191126180153-f0638e958b60 h1:gZucqLjL1eDzVWrXj4uiWeMbAopJlBR2mKQAsTGdPwo=
github.com/yuin/goldmark-meta v0.0.0-20191126180153-f0638e958b60/go.mod h1:i9VhcIHN2PxXMbQrKqXNueok6QNONoPjNMoj9MygVL0=
golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=


+ 79
- 18
main.go View File

@ -3,20 +3,36 @@ package main
import (
"bytes"
"fmt"
"github.com/tdewolff/minify/v2"
"github.com/tdewolff/minify/v2/css"
"github.com/tdewolff/minify/v2/html"
"html/template"
"io/ioutil"
"os"
"path"
"path/filepath"
"sort"
"strings"
"sync"
"time"
)
type Article struct {
Path string
Content string
Meta map[string]interface{}
Path string
SectionPath string
IsSection bool
Content template.HTML
Meta map[string]interface{}
SubArticles []*Article
}
func (article *Article) Title() string {
if article.Meta["title"] != nil {
if title, ok := (article.Meta["title"]).(string); ok {
return title
}
}
return ""
}
type parseResult struct {
@ -35,7 +51,9 @@ func main() {
}
fmt.Println("Finished parsing:", time.Now().String())
fmt.Println(len(articles), "articles")
err = generate(articles)
process(&articles)
fmt.Println("Finished processing:", time.Now().String())
err = generate(&articles)
if err != nil {
fmt.Println("Error:", err.Error())
return
@ -43,7 +61,8 @@ func main() {
fmt.Println("Finished generating:", time.Now().String())
}
func parse() ([]*Article, error) {
func parse() (map[string]*Article, error) {
// Parse files (meta and markdown
var files []string
_ = filepath.Walk("source", func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
@ -74,9 +93,9 @@ func parse() ([]*Article, error) {
return
}
// Compute article path
articlePath := strings.TrimPrefix(file, "source/")
articlePath := path.Clean(strings.TrimPrefix(file, "source/"))
if strings.HasSuffix(articlePath, "index.md") {
articlePath = strings.TrimSuffix(articlePath, "/index.md")
articlePath = path.Dir(articlePath)
} else {
articlePath = strings.TrimSuffix(articlePath, ".md")
}
@ -84,25 +103,59 @@ func parse() ([]*Article, error) {
resultsMutex.Lock()
results = append(results, parseResult{Article: &Article{
Path: articlePath,
Content: content,
Content: template.HTML(content),
Meta: metaData,
}})
resultsMutex.Unlock()
}()
}
waitGroup.Wait()
var articles []*Article
articles := map[string]*Article{}
for _, result := range results {
if result.Err != nil {
return nil, result.Err
} else {
articles = append(articles, result.Article)
articles[result.Article.Path] = result.Article
}
}
return articles, nil
}
func generate(articles []*Article) error {
func process(articles *map[string]*Article) {
for _, article := range *articles {
pathSegments := strings.Split(article.Path, "/")
// Set section
if len(pathSegments) > 1 && (*articles)[pathSegments[0]] != nil {
section := (*articles)[pathSegments[0]]
section.IsSection = true
article.SectionPath = section.Path
}
// Get cascaded meta
joinedSegments := ""
for _, segment := range pathSegments {
joinedSegments = path.Join(joinedSegments, segment)
if (*articles)[joinedSegments] != nil {
joinedSegmentsArticle := (*articles)[joinedSegments]
if cascade, ok := (joinedSegmentsArticle.Meta["cascade"]).(map[string]interface{}); ok {
for key, value := range cascade {
if article.Meta[key] == nil {
article.Meta[key] = value
}
}
}
if joinedSegments != article.Path {
joinedSegmentsArticle.SubArticles = append(joinedSegmentsArticle.SubArticles, article)
sort.Slice(joinedSegmentsArticle.SubArticles, func(i, j int) bool {
return joinedSegmentsArticle.SubArticles[i].Title() < joinedSegmentsArticle.SubArticles[j].Title()
})
}
}
}
}
return
}
func generate(articles *map[string]*Article) error {
// Delete old files
err := os.RemoveAll("output")
if err != nil {
@ -111,7 +164,10 @@ func generate(articles []*Article) error {
// Generate new files
var wg sync.WaitGroup
templates := template.Must(template.New("").ParseFiles("templates/single.gohtml"))
for _, article := range articles {
minifier := minify.New()
minifier.AddFunc("text/html", html.Minify)
minifier.AddFunc("text/css", css.Minify)
for _, article := range *articles {
article := article
wg.Add(1)
go func() {
@ -119,23 +175,28 @@ func generate(articles []*Article) error {
// Execute template
var result bytes.Buffer
err = templates.ExecuteTemplate(&result, "single.gohtml", struct {
Article *Article
Content template.HTML
Article *Article
AllArticles *map[string]*Article
}{
Article: article,
Content: template.HTML(article.Content),
Article: article,
AllArticles: articles,
})
if err != nil {
panic(err)
}
// Minify
minified, err := minifier.Bytes("text/html", result.Bytes())
if err != nil {
panic(err)
}
// Create directories
err := os.MkdirAll(path.Join("output", article.Path), os.ModePerm)
err = os.MkdirAll(path.Join("output", article.Path), os.ModePerm)
if err != nil {
panic(err)
}
// Write result
fsMutex.Lock()
err = ioutil.WriteFile(path.Join("output", article.Path, "index.html"), result.Bytes(), os.ModePerm)
err = ioutil.WriteFile(path.Join("output", article.Path, "index.html"), minified, os.ModePerm)
fsMutex.Unlock()
if err != nil {
panic(err)


+ 1
- 1
markdown.go View File

@ -32,7 +32,7 @@ func convert(source []byte) (content string, metaData map[string]interface{}, er
context := parser.NewContext()
var buffer bytes.Buffer
err = markdown.Convert(source, &buffer, parser.WithContext(context))
content = buffer.String()
content = string(emojify(buffer.Bytes()))
metaData = meta.Get(context)
return
}

+ 9
- 1
templates/single.gohtml View File

@ -85,6 +85,14 @@
<main class=h-entry>
<article>
<h1 class=p-name>{{ index .Article.Meta "title" }}</h1>
<div class=e-content>{{ .Content }}</div>
{{ if .Article.SectionPath }}
{{ with ( index .AllArticles .Article.SectionPath ) }}
<p><a href="/{{ .Path }}">{{ .Title }}</a></p>
{{ end }}
{{ end }}
<div class=e-content>{{ .Article.Content }}</div>
</article>
{{ range .Article.SubArticles }}
<p><a href="/{{ .Path }}">{{ .Title }}</a></p>
{{ end }}
</main>

Loading…
Cancel
Save