Rewrite to new yaml format

main
Antonio De Lucreziis 2 years ago
parent 9ebc4715ed
commit 46715cf033

3
.gitignore vendored

@ -1,4 +1,7 @@
.env
*.local* *.local*
bin/ bin/
dist/ dist/
.vscode/

@ -1,6 +1,20 @@
package cabret package cabret
const MatchResult = "MatchResult" import (
"fmt"
"log"
)
func init() {
log.SetFlags(0)
}
const MatchResultKey = "MatchResult"
type MatchResult struct {
File string
Captures map[string]string
}
type Map map[string]any type Map map[string]any
@ -20,21 +34,29 @@ type Content struct {
Metadata Map Metadata Map
} }
type Operation interface {
Load(config map[string]any) error
}
type ListOperation interface { type ListOperation interface {
MapAll(contents []Content) ([]Content, error) Operation
ProcessList(contents []Content) ([]Content, error)
} }
type ItemOperation interface { type ItemOperation interface {
FlatMap(content Content) (*Content, error) Operation
ProcessItem(content Content) (*Content, error)
} }
type FlatMapToMapAll struct{ FlatMapOperation } func ProcessOperation(op Operation, inputs []Content) ([]Content, error) {
switch op := op.(type) {
func (op FlatMapToMapAll) MapAll(contents []Content) ([]Content, error) { case ListOperation:
mapped := []Content{} return op.ProcessList(inputs)
for _, item := range contents { case ItemOperation:
result, err := op.FlatMap(item) outputs := []Content{}
for _, item := range inputs {
result, err := op.ProcessItem(item)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -43,9 +65,10 @@ func (op FlatMapToMapAll) MapAll(contents []Content) ([]Content, error) {
if result == nil { if result == nil {
continue continue
} }
outputs = append(outputs, *result)
mapped = append(mapped, *result) }
return outputs, nil
default:
return nil, fmt.Errorf(`invalid operation type %T`, op)
} }
return mapped, nil
} }

@ -9,17 +9,15 @@ import (
func main() { func main() {
log.SetFlags(0) log.SetFlags(0)
log.Printf("Rendering current project")
site, err := config.ReadCabretfile("./Cabretfile.yaml") cabretfile, err := config.ReadCabretfile("./Cabretfile.yaml")
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
// repr.Println(site) // repr.Println(cabretfile)
if err := exec.Execute(site); err != nil { if err := exec.Execute(cabretfile); err != nil {
log.Fatal(err) log.Fatal(err)
} }
} }

@ -6,24 +6,21 @@ import (
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
// Operation is an enum of various operations // Operation should have at least one key in "source", "use", "target". The remaining keys are options for that operation
type Operation map[string]any type Operation map[string]any
type EntryPoint struct { type Pipeline struct {
Source string `yaml:",omitempty"` Pipeline []Operation `yaml:"pipeline"`
Pipeline []Operation `yaml:",omitempty"`
} }
type Options struct { type BuildOptions struct {
// Excludes lists files and folders to globally exclude from compilation
Excludes []string `yaml:",omitempty"` Excludes []string `yaml:",omitempty"`
// Include []string `yaml:",omitempty"`
Output string `yaml:",omitempty"`
} }
// Cabretfile has some configuration for the
type Cabretfile struct { type Cabretfile struct {
Options Options `yaml:",omitempty"` Options BuildOptions
EntryPoints []*EntryPoint `yaml:"entryPoints"` Build []Pipeline
} }
func ReadCabretfile(file string) (*Cabretfile, error) { func ReadCabretfile(file string) (*Cabretfile, error) {

@ -1,12 +1,14 @@
entryPoints: build:
- pipeline:
- source: index.html - source: index.html
pipeline: - use: layout
- layout: layouts/base.html path: layouts/base.html
- target: dist/index.html - target: dist/index.html
- pipeline:
- source: posts/{id}.md - source: posts/{id}.md
pipeline: - use: markdown
- plugin: markdown - use: layout
- layout: layouts/base.html path: layouts/base.html
- target: dist/posts/{id}/index.html - target: dist/posts/{id}/index.html

@ -1,5 +1,6 @@
<h1>My Website</h1> <h1>My Website</h1>
<p> <p>
Lorem ipsum dolor sit amet consectetur adipisicing elit. Minima, eveniet, dolorum amet, cupiditate quae excepturi aspernatur dolor voluptatem obcaecati ratione quas? Et explicabo illum iure eius porro, dolor quos doloremque! Lorem ipsum dolor sit amet consectetur adipisicing elit. Minima, eveniet, dolorum amet, cupiditate quae excepturi aspernatur dolor
voluptatem obcaecati ratione quas? Et explicabo illum iure eius porro, dolor quos doloremque!
</p> </p>

@ -1,124 +1,21 @@
package exec package exec
import ( import (
"log"
"mime"
"os"
gopath "path"
"github.com/alecthomas/repr"
"github.com/aziis98/cabret" "github.com/aziis98/cabret"
"github.com/aziis98/cabret/config" "github.com/aziis98/cabret/config"
"github.com/aziis98/cabret/operation" "github.com/aziis98/cabret/pipeline"
"github.com/aziis98/cabret/path"
) )
type matchResult struct {
file string
context map[string]string
}
func BuildOperation(op config.Operation) cabret.Operation {
if path, ok := op["read"]; ok {
return cabret.FlatMapToMapAll{&operation.Target{
PathTemplate: path.(string),
}}
}
if path, ok := op["write"]; ok {
return cabret.FlatMapToMapAll{&operation.Target{
PathTemplate: path.(string),
}}
}
if name, ok := op["plugin"]; ok {
switch name {
case "layout":
path := op["path"].(string)
delete(op, "path")
return cabret.FlatMapToMapAll{&operation.Layout{
TemplateFilesPattern: path,
Options: op,
}}
case "markdown":
return cabret.FlatMapToMapAll{&operation.Markdown{
Options: op,
}}
default:
log.Fatalf(`invalid operation: %s`, name)
}
}
return nil
}
func Execute(cfg *config.Cabretfile) error { func Execute(cfg *config.Cabretfile) error {
files, err := cabret.FindFiles([]string{}) for _, p := range cfg.Build {
ops, err := pipeline.Parse(p)
if err != nil { if err != nil {
return err return err
} }
// the first index is the entrypoint ID, the second is for the array of matched files for this entrypoint if _, err := pipeline.Process([]cabret.Content{}, ops); err != nil {
entryPointMatches := make([][]matchResult, len(cfg.EntryPoints))
// load all files to process
for id, ep := range cfg.EntryPoints {
pat, err := path.ParsePattern(ep.Source)
if err != nil {
return err return err
} }
matchedFiles := []matchResult{}
for _, f := range files {
if ok, ctx, _ := pat.Match(f); ok {
log.Printf(`[Preload] [EntryPoint %d] Found "%s": %#v`, id, f, ctx)
matchedFiles = append(matchedFiles, matchResult{f, ctx})
}
}
entryPointMatches[id] = matchedFiles
}
// TODO: preload all metadata...
// process all entrypoints
for id, ep := range cfg.EntryPoints {
log.Printf(`[EntryPoint %d] starting to process %d file(s)`, id, len(entryPointMatches[id]))
for _, m := range entryPointMatches[id] {
log.Printf(`[EntryPoint %d] ["%s"] reading file`, id, m.file)
data, err := os.ReadFile(m.file)
if err != nil {
return err
}
content := cabret.Content{
Type: mime.TypeByExtension(gopath.Ext(m.file)),
Data: data,
Metadata: cabret.Map{
cabret.MatchResult: m.context,
},
}
for i, opConfig := range ep.Pipeline {
op := BuildOperation(opConfig)
log.Printf(`[EntryPoint %d] ["%s"] [Operation(%d)] applying %s`, id, m.file, i, repr.String(op))
newContent, err := op.Process(content)
if err != nil {
return err
}
if newContent == nil {
break
}
// log.Printf(`[EntryPoint %d] ["%s"] [Operation(%d)] [Metadata] %s`, id, m.file, i, repr.String(newContent.Metadata, repr.Indent(" ")))
content = *newContent
}
log.Printf(`[EntryPoint %d] ["%s"] done`, id, m.file)
}
} }
return nil return nil

@ -0,0 +1,39 @@
package operation
import (
"fmt"
"github.com/aziis98/cabret"
)
func init() {
registerType("categorize", &Categorize{})
}
type Categorize struct {
Key string
// Operation to be executed for each category
Operation cabret.Operation
}
func (op *Categorize) Load(config map[string]any) error {
{
v, ok := config["key"]
if !ok {
return fmt.Errorf(`missing "key" field`)
}
key, ok := v.(string)
if !ok {
return fmt.Errorf(`expected string but got "%v" of type %T`, v, v)
}
op.Key = key
}
return nil
}
func (op *Categorize) Process(content cabret.Content) (*cabret.Content, error) {
return nil, nil
}

@ -1,12 +0,0 @@
package operation
import "github.com/aziis98/cabret"
type GroupBy struct {
Key string
}
func (op GroupBy) Process(content cabret.Content) (*cabret.Content, error) {
return nil, nil
}

@ -1,38 +1,73 @@
package operation package operation
import ( import (
"html/template" "fmt"
goHtmlTemplate "html/template"
"log" "log"
"mime" "mime"
"path/filepath" "path/filepath"
"strings" "strings"
gopath "path"
"github.com/alecthomas/repr"
"github.com/aziis98/cabret" "github.com/aziis98/cabret"
"github.com/aziis98/cabret/operation/layout" "github.com/aziis98/cabret/operation/template"
"github.com/aziis98/cabret/util" "github.com/aziis98/cabret/util"
) )
var HtmlMimeType = mime.TypeByExtension(".html") var HtmlMimeType = mime.TypeByExtension(".html")
var _ cabret.FlatMapOperation = Layout{} func init() {
registerType("layout", &Layout{})
}
type Layout struct { type Layout struct {
// TemplateFilesPattern is a comma separated list of unix glob patterns // TemplatePatterns is a list of glob patterns of templates that will be loaded
TemplateFilesPattern string TemplatePatterns []string
Options map[string]any
} }
func (op Layout) FlatMap(content cabret.Content) (*cabret.Content, error) { func (op *Layout) Load(config map[string]any) error {
var tmpl layout.Template if v, ok := config[ShortFormValueKey]; ok {
globPatternsStr, ok := v.(string)
if !ok {
return fmt.Errorf(`expected a comma separated list of glob patterns but got "%v" of type %T`, v, v)
}
patterns := strings.Split(op.TemplateFilesPattern, ",") globPatterns := strings.Split(globPatternsStr, ",")
for _, pat := range globPatterns {
op.TemplatePatterns = append(op.TemplatePatterns, strings.TrimSpace(pat))
}
return nil
}
if v, ok := config["paths"]; ok {
globPatterns, ok := v.([]string)
if !ok {
return fmt.Errorf(`expected a list of glob patterns but got "%v" of type %T`, v, v)
}
for _, pat := range globPatterns {
op.TemplatePatterns = append(op.TemplatePatterns, strings.TrimSpace(pat))
}
return nil
}
if v, ok := config["path"]; ok {
globPatternStr, ok := v.(string)
if !ok {
return fmt.Errorf(`expected a glob pattern but got "%v" of type %T`, v, v)
}
op.TemplatePatterns = []string{strings.TrimSpace(globPatternStr)}
return nil
}
return fmt.Errorf(`invalid config`)
}
func (op Layout) ProcessItem(content cabret.Content) (*cabret.Content, error) {
// expand glob patterns
tmplFiles := []string{} tmplFiles := []string{}
for _, pat := range patterns { for _, pat := range op.TemplatePatterns {
files, err := filepath.Glob(strings.TrimSpace(pat)) files, err := filepath.Glob(strings.TrimSpace(pat))
if err != nil { if err != nil {
return nil, err return nil, err
@ -41,28 +76,22 @@ func (op Layout) FlatMap(content cabret.Content) (*cabret.Content, error) {
tmplFiles = append(tmplFiles, files...) tmplFiles = append(tmplFiles, files...)
} }
log.Printf(`[Layout] template pattern "%s" expanded to %s`, op.TemplateFilesPattern, repr.String(tmplFiles)) // create template
tmpl, err := template.ParseFiles(tmplFiles...)
if gopath.Ext(tmplFiles[0]) == ".html" { if err != nil {
var err error
if tmpl, err = layout.NewHtmlTemplate(tmplFiles...); err != nil {
return nil, err
}
} else {
var err error
if tmpl, err = layout.NewTextTemplate(tmplFiles...); err != nil {
return nil, err return nil, err
} }
}
ctx := util.CloneMap(content.Metadata) ctx := util.CloneMap(content.Metadata)
if content.Type == HtmlMimeType { if content.Type == HtmlMimeType {
ctx["Content"] = template.HTML(content.Data) ctx["Content"] = goHtmlTemplate.HTML(content.Data)
} else { } else {
ctx["Content"] = content.Data ctx["Content"] = content.Data
} }
log.Printf(`[operation.Layout] rendering into layout "%s"`, strings.Join(op.TemplatePatterns, ", "))
data, err := tmpl.Render(ctx) data, err := tmpl.Render(ctx)
if err != nil { if err != nil {
return nil, err return nil, err

@ -1,5 +0,0 @@
package layout
type Template interface {
Render(ctx map[string]any) ([]byte, error)
}

@ -2,6 +2,7 @@ package operation
import ( import (
"bytes" "bytes"
"log"
"github.com/aziis98/cabret" "github.com/aziis98/cabret"
"github.com/iancoleman/strcase" "github.com/iancoleman/strcase"
@ -11,13 +12,19 @@ import (
"github.com/yuin/goldmark/parser" "github.com/yuin/goldmark/parser"
) )
var _ cabret.FlatMapOperation = Markdown{} func init() {
registerType("markdown", &Markdown{})
}
type Markdown struct { type Markdown struct {
Options map[string]any Options map[string]any
} }
func (op Markdown) FlatMap(content cabret.Content) (*cabret.Content, error) { func (op *Markdown) Load(config map[string]any) error {
return nil
}
func (op Markdown) ProcessItem(content cabret.Content) (*cabret.Content, error) {
md := goldmark.New( md := goldmark.New(
goldmark.WithExtensions( goldmark.WithExtensions(
extension.GFM, extension.GFM,
@ -30,6 +37,8 @@ func (op Markdown) FlatMap(content cabret.Content) (*cabret.Content, error) {
var buf bytes.Buffer var buf bytes.Buffer
log.Printf(`[operation.Markdown] rendering markdown`)
context := parser.NewContext() context := parser.NewContext()
if err := md.Convert(content.Data, &buf, parser.WithContext(context)); err != nil { if err := md.Convert(content.Data, &buf, parser.WithContext(context)); err != nil {
panic(err) panic(err)

@ -1,40 +0,0 @@
package operation
import (
"mime"
"os"
gopath "path"
"path/filepath"
"github.com/aziis98/cabret"
)
var _ cabret.ListOperation = Read{}
type Read struct {
Patterns []string
}
func (op Read) MapAll(contents []cabret.Content) ([]cabret.Content, error) {
for _, pattern := range op.Patterns {
files, err := filepath.Glob(pattern)
if err != nil {
return nil, err
}
for _, file := range files {
data, err := os.ReadFile(file)
if err != nil {
return nil, err
}
contents = append(contents, cabret.Content{
Type: mime.TypeByExtension(gopath.Ext(file)),
Data: data,
Metadata: cabret.Map{},
})
}
}
return contents, nil
}

@ -0,0 +1,34 @@
package operation
import (
"fmt"
"log"
"reflect"
"github.com/aziis98/cabret"
)
// ShortFormValueKey is used by some operations that support an inline form
const ShortFormValueKey = "value"
var registry = map[string]reflect.Type{}
func registerType(name string, op cabret.Operation) {
typ := reflect.TypeOf(op).Elem()
log.Printf(`[operation] registered type "%v"`, typ)
registry[name] = typ
}
func Build(name string, options map[string]any) (cabret.Operation, error) {
typ, ok := registry[name]
if !ok {
return nil, fmt.Errorf(`no registered operation named %q`, name)
}
op := reflect.New(typ).Interface().(cabret.Operation)
if err := op.Load(options); err != nil {
return nil, err
}
return op, nil
}

@ -0,0 +1,18 @@
package operation
import (
"testing"
"gotest.tools/assert"
)
func TestBuild(t *testing.T) {
op, err := Build("categorize", map[string]any{
"key": "tags",
})
assert.NilError(t, err)
assert.DeepEqual(t, op, &Categorize{
Key: "tags",
})
}

@ -0,0 +1,88 @@
package operation
import (
"fmt"
"log"
"mime"
"os"
gopath "path"
"github.com/aziis98/cabret"
"github.com/aziis98/cabret/path"
)
func init() {
registerType("source", &Source{})
}
// Source is a ListOperation that appends the matched files to the processing items
type Source struct {
Patterns []string
}
func (op *Source) Load(config map[string]any) error {
if v, ok := config[ShortFormValueKey]; ok {
pattern, ok := v.(string)
if !ok {
return fmt.Errorf(`expected pattern but got "%v" of type %T`, v, v)
}
op.Patterns = []string{pattern}
return nil
}
if v, ok := config["paths"]; ok {
patterns, ok := v.([]string)
if !ok {
return fmt.Errorf(`expected list of patterns but got "%v" of type %T`, v, v)
}
op.Patterns = patterns
return nil
}
return fmt.Errorf(`invalid config for "source": %#v`, config)
}
func (op Source) ProcessList(contents []cabret.Content) ([]cabret.Content, error) {
files, err := cabret.FindFiles([]string{})
if err != nil {
return nil, err
}
matches := []cabret.MatchResult{}
for _, patternStr := range op.Patterns {
pat, err := path.ParsePattern(patternStr)
if err != nil {
return nil, err
}
for _, f := range files {
if ok, captures, _ := pat.Match(f); ok {
matches = append(matches, cabret.MatchResult{
File: f,
Captures: captures,
})
}
}
}
for _, m := range matches {
log.Printf(`[operation.Source] reading "%s"`, m.File)
data, err := os.ReadFile(m.File)
if err != nil {
return nil, err
}
contents = append(contents, cabret.Content{
Type: mime.TypeByExtension(gopath.Ext(m.File)),
Data: data,
Metadata: cabret.Map{
cabret.MatchResultKey: m.Captures,
},
})
}
return contents, nil
}

@ -2,6 +2,7 @@ package operation
import ( import (
"fmt" "fmt"
"log"
"os" "os"
gopath "path" gopath "path"
@ -10,20 +11,38 @@ import (
"github.com/aziis98/cabret/path" "github.com/aziis98/cabret/path"
) )
var _ cabret.FlatMapOperation = Target{} func init() {
registerType("target", &Target{})
}
type Target struct { type Target struct {
PathTemplate string PathTemplate string
} }
func (op Target) FlatMap(c cabret.Content) (*cabret.Content, error) { func (op *Target) Load(config map[string]any) error {
mr, ok := c.Metadata[cabret.MatchResult].(map[string]string) if v, ok := config[ShortFormValueKey]; ok {
template, ok := v.(string)
if !ok { if !ok {
return nil, fmt.Errorf(`invalid match result type %T`, c.Metadata[cabret.MatchResult]) return fmt.Errorf(`expected pattern but got "%v" of type %T`, v, v)
}
op.PathTemplate = template
return nil
}
return fmt.Errorf(`invalid config for "target": %#v`, config)
}
func (op Target) ProcessItem(c cabret.Content) (*cabret.Content, error) {
mr, ok := c.Metadata[cabret.MatchResultKey].(map[string]string)
if !ok {
return nil, fmt.Errorf(`invalid match result type %T`, c.Metadata[cabret.MatchResultKey])
} }
target := path.RenderTemplate(op.PathTemplate, mr) target := path.RenderTemplate(op.PathTemplate, mr)
log.Printf(`[operation.Target] writing "%s"`, target)
if err := os.MkdirAll(gopath.Dir(target), 0777); err != nil { if err := os.MkdirAll(gopath.Dir(target), 0777); err != nil {
return nil, err return nil, err
} }

@ -1,18 +1,18 @@
package layout package template
import ( import (
"bytes" "bytes"
"html/template" goHtmlTemplate "html/template"
) )
var _ Template = HtmlTemplate{} var _ Template = HtmlTemplate{}
type HtmlTemplate struct { type HtmlTemplate struct {
*template.Template *goHtmlTemplate.Template
} }
func NewHtmlTemplate(files ...string) (*HtmlTemplate, error) { func NewHtmlTemplate(files ...string) (*HtmlTemplate, error) {
t, err := template.ParseFiles(files...) t, err := goHtmlTemplate.ParseFiles(files...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -21,11 +21,10 @@ func NewHtmlTemplate(files ...string) (*HtmlTemplate, error) {
} }
func (t HtmlTemplate) Render(ctx map[string]any) ([]byte, error) { func (t HtmlTemplate) Render(ctx map[string]any) ([]byte, error) {
var b bytes.Buffer var buf bytes.Buffer
if err := t.Template.Execute(&buf, ctx); err != nil {
if err := t.Template.Execute(&b, ctx); err != nil {
return nil, err return nil, err
} }
return b.Bytes(), nil return buf.Bytes(), nil
} }

@ -0,0 +1,17 @@
package template
import (
"path/filepath"
)
type Template interface {
Render(ctx map[string]any) ([]byte, error)
}
func ParseFiles(files ...string) (Template, error) {
if filepath.Ext(files[0]) == ".html" {
return NewHtmlTemplate(files...)
}
return NewTextTemplate(files...)
}

@ -1,18 +1,18 @@
package layout package template
import ( import (
"bytes" "bytes"
"text/template" goTextTemplate "text/template"
) )
var _ Template = TextTemplate{} var _ Template = TextTemplate{}
type TextTemplate struct { type TextTemplate struct {
*template.Template *goTextTemplate.Template
} }
func NewTextTemplate(files ...string) (*TextTemplate, error) { func NewTextTemplate(files ...string) (*TextTemplate, error) {
t, err := template.ParseFiles(files...) t, err := goTextTemplate.ParseFiles(files...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -21,11 +21,10 @@ func NewTextTemplate(files ...string) (*TextTemplate, error) {
} }
func (t TextTemplate) Render(ctx map[string]any) ([]byte, error) { func (t TextTemplate) Render(ctx map[string]any) ([]byte, error) {
var b bytes.Buffer var buf bytes.Buffer
if err := t.Template.Execute(&buf, ctx); err != nil {
if err := t.Template.Execute(&b, ctx); err != nil {
return nil, err return nil, err
} }
return b.Bytes(), nil return buf.Bytes(), nil
} }

@ -0,0 +1,88 @@
package pipeline
import (
"fmt"
"github.com/aziis98/cabret"
"github.com/aziis98/cabret/config"
"github.com/aziis98/cabret/operation"
)
func switchMap(m map[string]any, v *any) func(k string) bool {
return func(k string) bool {
val, ok := m[k]
if ok {
*v = val
}
return ok
}
}
func Parse(p config.Pipeline) ([]cabret.Operation, error) {
ops := []cabret.Operation{}
for _, opConfig := range p.Pipeline {
var v any
has := switchMap(opConfig, &v)
switch {
case has("source"):
value, ok := v.(string)
if !ok {
return nil, fmt.Errorf(`expected string but got "%v" of type %T`, v, v)
}
opConfig[operation.ShortFormValueKey] = value
op := &operation.Source{}
if err := op.Load(opConfig); err != nil {
return nil, err
}
ops = append(ops, op)
case has("target"):
value, ok := v.(string)
if !ok {
return nil, fmt.Errorf(`expected string but got "%v" of type %T`, v, v)
}
opConfig[operation.ShortFormValueKey] = value
op := &operation.Target{}
if err := op.Load(opConfig); err != nil {
return nil, err
}
ops = append(ops, op)
case has("use"):
name, ok := v.(string)
if !ok {
return nil, fmt.Errorf(`expected string but got "%v" of type %T`, v, v)
}
op, err := operation.Build(name, opConfig)
if err != nil {
return nil, err
}
ops = append(ops, op)
default:
return nil, fmt.Errorf(`pipeline entry is missing one of "use", "source" or "target", got %#v`, opConfig)
}
}
return ops, nil
}
func Process(contents []cabret.Content, ops []cabret.Operation) ([]cabret.Content, error) {
for _, op := range ops {
var err error
contents, err = cabret.ProcessOperation(op, contents)
if err != nil {
return nil, err
}
}
return contents, nil
}
Loading…
Cancel
Save