Rewrite to new yaml format

main
Antonio De Lucreziis 2 years ago
parent 9ebc4715ed
commit 46715cf033

3
.gitignore vendored

@ -1,4 +1,7 @@
.env
*.local*
bin/
dist/
.vscode/

@ -1,6 +1,20 @@
package cabret
const MatchResult = "MatchResult"
import (
"fmt"
"log"
)
func init() {
log.SetFlags(0)
}
const MatchResultKey = "MatchResult"
type MatchResult struct {
File string
Captures map[string]string
}
type Map map[string]any
@ -20,21 +34,29 @@ type Content struct {
Metadata Map
}
type Operation interface {
Load(config map[string]any) error
}
type ListOperation interface {
MapAll(contents []Content) ([]Content, error)
Operation
ProcessList(contents []Content) ([]Content, error)
}
type ItemOperation interface {
FlatMap(content Content) (*Content, error)
Operation
ProcessItem(content Content) (*Content, error)
}
type FlatMapToMapAll struct{ FlatMapOperation }
func (op FlatMapToMapAll) MapAll(contents []Content) ([]Content, error) {
mapped := []Content{}
func ProcessOperation(op Operation, inputs []Content) ([]Content, error) {
switch op := op.(type) {
case ListOperation:
return op.ProcessList(inputs)
for _, item := range contents {
result, err := op.FlatMap(item)
case ItemOperation:
outputs := []Content{}
for _, item := range inputs {
result, err := op.ProcessItem(item)
if err != nil {
return nil, err
}
@ -43,9 +65,10 @@ func (op FlatMapToMapAll) MapAll(contents []Content) ([]Content, error) {
if result == nil {
continue
}
mapped = append(mapped, *result)
outputs = append(outputs, *result)
}
return outputs, nil
default:
return nil, fmt.Errorf(`invalid operation type %T`, op)
}
return mapped, nil
}

@ -9,17 +9,15 @@ import (
func main() {
log.SetFlags(0)
log.Printf("Rendering current project")
site, err := config.ReadCabretfile("./Cabretfile.yaml")
cabretfile, err := config.ReadCabretfile("./Cabretfile.yaml")
if err != nil {
log.Fatal(err)
}
// repr.Println(site)
// repr.Println(cabretfile)
if err := exec.Execute(site); err != nil {
if err := exec.Execute(cabretfile); err != nil {
log.Fatal(err)
}
}

@ -6,24 +6,21 @@ import (
"gopkg.in/yaml.v3"
)
// Operation is an enum of various operations
// Operation should have at least one key in "source", "use", "target". The remaining keys are options for that operation
type Operation map[string]any
type EntryPoint struct {
Source string `yaml:",omitempty"`
Pipeline []Operation `yaml:",omitempty"`
type Pipeline struct {
Pipeline []Operation `yaml:"pipeline"`
}
type Options struct {
type BuildOptions struct {
// Excludes lists files and folders to globally exclude from compilation
Excludes []string `yaml:",omitempty"`
// Include []string `yaml:",omitempty"`
Output string `yaml:",omitempty"`
}
// Cabretfile has some configuration for the
type Cabretfile struct {
Options Options `yaml:",omitempty"`
EntryPoints []*EntryPoint `yaml:"entryPoints"`
Options BuildOptions
Build []Pipeline
}
func ReadCabretfile(file string) (*Cabretfile, error) {

@ -1,12 +1,14 @@
entryPoints:
build:
- pipeline:
- source: index.html
pipeline:
- layout: layouts/base.html
- use: layout
path: layouts/base.html
- target: dist/index.html
- pipeline:
- source: posts/{id}.md
pipeline:
- plugin: markdown
- layout: layouts/base.html
- use: markdown
- use: layout
path: layouts/base.html
- target: dist/posts/{id}/index.html

@ -1,5 +1,6 @@
<h1>My Website</h1>
<p>
Lorem ipsum dolor sit amet consectetur adipisicing elit. Minima, eveniet, dolorum amet, cupiditate quae excepturi aspernatur dolor voluptatem obcaecati ratione quas? Et explicabo illum iure eius porro, dolor quos doloremque!
Lorem ipsum dolor sit amet consectetur adipisicing elit. Minima, eveniet, dolorum amet, cupiditate quae excepturi aspernatur dolor
voluptatem obcaecati ratione quas? Et explicabo illum iure eius porro, dolor quos doloremque!
</p>

@ -1,124 +1,21 @@
package exec
import (
"log"
"mime"
"os"
gopath "path"
"github.com/alecthomas/repr"
"github.com/aziis98/cabret"
"github.com/aziis98/cabret/config"
"github.com/aziis98/cabret/operation"
"github.com/aziis98/cabret/path"
"github.com/aziis98/cabret/pipeline"
)
type matchResult struct {
file string
context map[string]string
}
func BuildOperation(op config.Operation) cabret.Operation {
if path, ok := op["read"]; ok {
return cabret.FlatMapToMapAll{&operation.Target{
PathTemplate: path.(string),
}}
}
if path, ok := op["write"]; ok {
return cabret.FlatMapToMapAll{&operation.Target{
PathTemplate: path.(string),
}}
}
if name, ok := op["plugin"]; ok {
switch name {
case "layout":
path := op["path"].(string)
delete(op, "path")
return cabret.FlatMapToMapAll{&operation.Layout{
TemplateFilesPattern: path,
Options: op,
}}
case "markdown":
return cabret.FlatMapToMapAll{&operation.Markdown{
Options: op,
}}
default:
log.Fatalf(`invalid operation: %s`, name)
}
}
return nil
}
func Execute(cfg *config.Cabretfile) error {
files, err := cabret.FindFiles([]string{})
for _, p := range cfg.Build {
ops, err := pipeline.Parse(p)
if err != nil {
return err
}
// the first index is the entrypoint ID, the second is for the array of matched files for this entrypoint
entryPointMatches := make([][]matchResult, len(cfg.EntryPoints))
// load all files to process
for id, ep := range cfg.EntryPoints {
pat, err := path.ParsePattern(ep.Source)
if err != nil {
if _, err := pipeline.Process([]cabret.Content{}, ops); err != nil {
return err
}
matchedFiles := []matchResult{}
for _, f := range files {
if ok, ctx, _ := pat.Match(f); ok {
log.Printf(`[Preload] [EntryPoint %d] Found "%s": %#v`, id, f, ctx)
matchedFiles = append(matchedFiles, matchResult{f, ctx})
}
}
entryPointMatches[id] = matchedFiles
}
// TODO: preload all metadata...
// process all entrypoints
for id, ep := range cfg.EntryPoints {
log.Printf(`[EntryPoint %d] starting to process %d file(s)`, id, len(entryPointMatches[id]))
for _, m := range entryPointMatches[id] {
log.Printf(`[EntryPoint %d] ["%s"] reading file`, id, m.file)
data, err := os.ReadFile(m.file)
if err != nil {
return err
}
content := cabret.Content{
Type: mime.TypeByExtension(gopath.Ext(m.file)),
Data: data,
Metadata: cabret.Map{
cabret.MatchResult: m.context,
},
}
for i, opConfig := range ep.Pipeline {
op := BuildOperation(opConfig)
log.Printf(`[EntryPoint %d] ["%s"] [Operation(%d)] applying %s`, id, m.file, i, repr.String(op))
newContent, err := op.Process(content)
if err != nil {
return err
}
if newContent == nil {
break
}
// log.Printf(`[EntryPoint %d] ["%s"] [Operation(%d)] [Metadata] %s`, id, m.file, i, repr.String(newContent.Metadata, repr.Indent(" ")))
content = *newContent
}
log.Printf(`[EntryPoint %d] ["%s"] done`, id, m.file)
}
}
return nil

@ -0,0 +1,39 @@
package operation
import (
"fmt"
"github.com/aziis98/cabret"
)
func init() {
registerType("categorize", &Categorize{})
}
type Categorize struct {
Key string
// Operation to be executed for each category
Operation cabret.Operation
}
func (op *Categorize) Load(config map[string]any) error {
{
v, ok := config["key"]
if !ok {
return fmt.Errorf(`missing "key" field`)
}
key, ok := v.(string)
if !ok {
return fmt.Errorf(`expected string but got "%v" of type %T`, v, v)
}
op.Key = key
}
return nil
}
func (op *Categorize) Process(content cabret.Content) (*cabret.Content, error) {
return nil, nil
}

@ -1,12 +0,0 @@
package operation
import "github.com/aziis98/cabret"
type GroupBy struct {
Key string
}
func (op GroupBy) Process(content cabret.Content) (*cabret.Content, error) {
return nil, nil
}

@ -1,38 +1,73 @@
package operation
import (
"html/template"
"fmt"
goHtmlTemplate "html/template"
"log"
"mime"
"path/filepath"
"strings"
gopath "path"
"github.com/alecthomas/repr"
"github.com/aziis98/cabret"
"github.com/aziis98/cabret/operation/layout"
"github.com/aziis98/cabret/operation/template"
"github.com/aziis98/cabret/util"
)
var HtmlMimeType = mime.TypeByExtension(".html")
var _ cabret.FlatMapOperation = Layout{}
func init() {
registerType("layout", &Layout{})
}
type Layout struct {
// TemplateFilesPattern is a comma separated list of unix glob patterns
TemplateFilesPattern string
// TemplatePatterns is a list of glob patterns of templates that will be loaded
TemplatePatterns []string
}
func (op *Layout) Load(config map[string]any) error {
if v, ok := config[ShortFormValueKey]; ok {
globPatternsStr, ok := v.(string)
if !ok {
return fmt.Errorf(`expected a comma separated list of glob patterns but got "%v" of type %T`, v, v)
}
globPatterns := strings.Split(globPatternsStr, ",")
for _, pat := range globPatterns {
op.TemplatePatterns = append(op.TemplatePatterns, strings.TrimSpace(pat))
}
return nil
}
if v, ok := config["paths"]; ok {
globPatterns, ok := v.([]string)
if !ok {
return fmt.Errorf(`expected a list of glob patterns but got "%v" of type %T`, v, v)
}
Options map[string]any
for _, pat := range globPatterns {
op.TemplatePatterns = append(op.TemplatePatterns, strings.TrimSpace(pat))
}
func (op Layout) FlatMap(content cabret.Content) (*cabret.Content, error) {
var tmpl layout.Template
return nil
}
if v, ok := config["path"]; ok {
globPatternStr, ok := v.(string)
if !ok {
return fmt.Errorf(`expected a glob pattern but got "%v" of type %T`, v, v)
}
patterns := strings.Split(op.TemplateFilesPattern, ",")
op.TemplatePatterns = []string{strings.TrimSpace(globPatternStr)}
return nil
}
return fmt.Errorf(`invalid config`)
}
func (op Layout) ProcessItem(content cabret.Content) (*cabret.Content, error) {
// expand glob patterns
tmplFiles := []string{}
for _, pat := range patterns {
for _, pat := range op.TemplatePatterns {
files, err := filepath.Glob(strings.TrimSpace(pat))
if err != nil {
return nil, err
@ -41,28 +76,22 @@ func (op Layout) FlatMap(content cabret.Content) (*cabret.Content, error) {
tmplFiles = append(tmplFiles, files...)
}
log.Printf(`[Layout] template pattern "%s" expanded to %s`, op.TemplateFilesPattern, repr.String(tmplFiles))
if gopath.Ext(tmplFiles[0]) == ".html" {
var err error
if tmpl, err = layout.NewHtmlTemplate(tmplFiles...); err != nil {
return nil, err
}
} else {
var err error
if tmpl, err = layout.NewTextTemplate(tmplFiles...); err != nil {
// create template
tmpl, err := template.ParseFiles(tmplFiles...)
if err != nil {
return nil, err
}
}
ctx := util.CloneMap(content.Metadata)
if content.Type == HtmlMimeType {
ctx["Content"] = template.HTML(content.Data)
ctx["Content"] = goHtmlTemplate.HTML(content.Data)
} else {
ctx["Content"] = content.Data
}
log.Printf(`[operation.Layout] rendering into layout "%s"`, strings.Join(op.TemplatePatterns, ", "))
data, err := tmpl.Render(ctx)
if err != nil {
return nil, err

@ -1,5 +0,0 @@
package layout
type Template interface {
Render(ctx map[string]any) ([]byte, error)
}

@ -2,6 +2,7 @@ package operation
import (
"bytes"
"log"
"github.com/aziis98/cabret"
"github.com/iancoleman/strcase"
@ -11,13 +12,19 @@ import (
"github.com/yuin/goldmark/parser"
)
var _ cabret.FlatMapOperation = Markdown{}
func init() {
registerType("markdown", &Markdown{})
}
type Markdown struct {
Options map[string]any
}
func (op Markdown) FlatMap(content cabret.Content) (*cabret.Content, error) {
func (op *Markdown) Load(config map[string]any) error {
return nil
}
func (op Markdown) ProcessItem(content cabret.Content) (*cabret.Content, error) {
md := goldmark.New(
goldmark.WithExtensions(
extension.GFM,
@ -30,6 +37,8 @@ func (op Markdown) FlatMap(content cabret.Content) (*cabret.Content, error) {
var buf bytes.Buffer
log.Printf(`[operation.Markdown] rendering markdown`)
context := parser.NewContext()
if err := md.Convert(content.Data, &buf, parser.WithContext(context)); err != nil {
panic(err)

@ -1,40 +0,0 @@
package operation
import (
"mime"
"os"
gopath "path"
"path/filepath"
"github.com/aziis98/cabret"
)
var _ cabret.ListOperation = Read{}
type Read struct {
Patterns []string
}
func (op Read) MapAll(contents []cabret.Content) ([]cabret.Content, error) {
for _, pattern := range op.Patterns {
files, err := filepath.Glob(pattern)
if err != nil {
return nil, err
}
for _, file := range files {
data, err := os.ReadFile(file)
if err != nil {
return nil, err
}
contents = append(contents, cabret.Content{
Type: mime.TypeByExtension(gopath.Ext(file)),
Data: data,
Metadata: cabret.Map{},
})
}
}
return contents, nil
}

@ -0,0 +1,34 @@
package operation
import (
"fmt"
"log"
"reflect"
"github.com/aziis98/cabret"
)
// ShortFormValueKey is used by some operations that support an inline form
const ShortFormValueKey = "value"
var registry = map[string]reflect.Type{}
func registerType(name string, op cabret.Operation) {
typ := reflect.TypeOf(op).Elem()
log.Printf(`[operation] registered type "%v"`, typ)
registry[name] = typ
}
func Build(name string, options map[string]any) (cabret.Operation, error) {
typ, ok := registry[name]
if !ok {
return nil, fmt.Errorf(`no registered operation named %q`, name)
}
op := reflect.New(typ).Interface().(cabret.Operation)
if err := op.Load(options); err != nil {
return nil, err
}
return op, nil
}

@ -0,0 +1,18 @@
package operation
import (
"testing"
"gotest.tools/assert"
)
func TestBuild(t *testing.T) {
op, err := Build("categorize", map[string]any{
"key": "tags",
})
assert.NilError(t, err)
assert.DeepEqual(t, op, &Categorize{
Key: "tags",
})
}

@ -0,0 +1,88 @@
package operation
import (
"fmt"
"log"
"mime"
"os"
gopath "path"
"github.com/aziis98/cabret"
"github.com/aziis98/cabret/path"
)
func init() {
registerType("source", &Source{})
}
// Source is a ListOperation that appends the matched files to the processing items
type Source struct {
Patterns []string
}
func (op *Source) Load(config map[string]any) error {
if v, ok := config[ShortFormValueKey]; ok {
pattern, ok := v.(string)
if !ok {
return fmt.Errorf(`expected pattern but got "%v" of type %T`, v, v)
}
op.Patterns = []string{pattern}
return nil
}
if v, ok := config["paths"]; ok {
patterns, ok := v.([]string)
if !ok {
return fmt.Errorf(`expected list of patterns but got "%v" of type %T`, v, v)
}
op.Patterns = patterns
return nil
}
return fmt.Errorf(`invalid config for "source": %#v`, config)
}
func (op Source) ProcessList(contents []cabret.Content) ([]cabret.Content, error) {
files, err := cabret.FindFiles([]string{})
if err != nil {
return nil, err
}
matches := []cabret.MatchResult{}
for _, patternStr := range op.Patterns {
pat, err := path.ParsePattern(patternStr)
if err != nil {
return nil, err
}
for _, f := range files {
if ok, captures, _ := pat.Match(f); ok {
matches = append(matches, cabret.MatchResult{
File: f,
Captures: captures,
})
}
}
}
for _, m := range matches {
log.Printf(`[operation.Source] reading "%s"`, m.File)
data, err := os.ReadFile(m.File)
if err != nil {
return nil, err
}
contents = append(contents, cabret.Content{
Type: mime.TypeByExtension(gopath.Ext(m.File)),
Data: data,
Metadata: cabret.Map{
cabret.MatchResultKey: m.Captures,
},
})
}
return contents, nil
}

@ -2,6 +2,7 @@ package operation
import (
"fmt"
"log"
"os"
gopath "path"
@ -10,20 +11,38 @@ import (
"github.com/aziis98/cabret/path"
)
var _ cabret.FlatMapOperation = Target{}
func init() {
registerType("target", &Target{})
}
type Target struct {
PathTemplate string
}
func (op Target) FlatMap(c cabret.Content) (*cabret.Content, error) {
mr, ok := c.Metadata[cabret.MatchResult].(map[string]string)
func (op *Target) Load(config map[string]any) error {
if v, ok := config[ShortFormValueKey]; ok {
template, ok := v.(string)
if !ok {
return nil, fmt.Errorf(`invalid match result type %T`, c.Metadata[cabret.MatchResult])
return fmt.Errorf(`expected pattern but got "%v" of type %T`, v, v)
}
op.PathTemplate = template
return nil
}
return fmt.Errorf(`invalid config for "target": %#v`, config)
}
func (op Target) ProcessItem(c cabret.Content) (*cabret.Content, error) {
mr, ok := c.Metadata[cabret.MatchResultKey].(map[string]string)
if !ok {
return nil, fmt.Errorf(`invalid match result type %T`, c.Metadata[cabret.MatchResultKey])
}
target := path.RenderTemplate(op.PathTemplate, mr)
log.Printf(`[operation.Target] writing "%s"`, target)
if err := os.MkdirAll(gopath.Dir(target), 0777); err != nil {
return nil, err
}

@ -1,18 +1,18 @@
package layout
package template
import (
"bytes"
"html/template"
goHtmlTemplate "html/template"
)
var _ Template = HtmlTemplate{}
type HtmlTemplate struct {
*template.Template
*goHtmlTemplate.Template
}
func NewHtmlTemplate(files ...string) (*HtmlTemplate, error) {
t, err := template.ParseFiles(files...)
t, err := goHtmlTemplate.ParseFiles(files...)
if err != nil {
return nil, err
}
@ -21,11 +21,10 @@ func NewHtmlTemplate(files ...string) (*HtmlTemplate, error) {
}
func (t HtmlTemplate) Render(ctx map[string]any) ([]byte, error) {
var b bytes.Buffer
if err := t.Template.Execute(&b, ctx); err != nil {
var buf bytes.Buffer
if err := t.Template.Execute(&buf, ctx); err != nil {
return nil, err
}
return b.Bytes(), nil
return buf.Bytes(), nil
}

@ -0,0 +1,17 @@
package template
import (
"path/filepath"
)
type Template interface {
Render(ctx map[string]any) ([]byte, error)
}
func ParseFiles(files ...string) (Template, error) {
if filepath.Ext(files[0]) == ".html" {
return NewHtmlTemplate(files...)
}
return NewTextTemplate(files...)
}

@ -1,18 +1,18 @@
package layout
package template
import (
"bytes"
"text/template"
goTextTemplate "text/template"
)
var _ Template = TextTemplate{}
type TextTemplate struct {
*template.Template
*goTextTemplate.Template
}
func NewTextTemplate(files ...string) (*TextTemplate, error) {
t, err := template.ParseFiles(files...)
t, err := goTextTemplate.ParseFiles(files...)
if err != nil {
return nil, err
}
@ -21,11 +21,10 @@ func NewTextTemplate(files ...string) (*TextTemplate, error) {
}
func (t TextTemplate) Render(ctx map[string]any) ([]byte, error) {
var b bytes.Buffer
if err := t.Template.Execute(&b, ctx); err != nil {
var buf bytes.Buffer
if err := t.Template.Execute(&buf, ctx); err != nil {
return nil, err
}
return b.Bytes(), nil
return buf.Bytes(), nil
}

@ -0,0 +1,88 @@
package pipeline
import (
"fmt"
"github.com/aziis98/cabret"
"github.com/aziis98/cabret/config"
"github.com/aziis98/cabret/operation"
)
func switchMap(m map[string]any, v *any) func(k string) bool {
return func(k string) bool {
val, ok := m[k]
if ok {
*v = val
}
return ok
}
}
func Parse(p config.Pipeline) ([]cabret.Operation, error) {
ops := []cabret.Operation{}
for _, opConfig := range p.Pipeline {
var v any
has := switchMap(opConfig, &v)
switch {
case has("source"):
value, ok := v.(string)
if !ok {
return nil, fmt.Errorf(`expected string but got "%v" of type %T`, v, v)
}
opConfig[operation.ShortFormValueKey] = value
op := &operation.Source{}
if err := op.Load(opConfig); err != nil {
return nil, err
}
ops = append(ops, op)
case has("target"):
value, ok := v.(string)
if !ok {
return nil, fmt.Errorf(`expected string but got "%v" of type %T`, v, v)
}
opConfig[operation.ShortFormValueKey] = value
op := &operation.Target{}
if err := op.Load(opConfig); err != nil {
return nil, err
}
ops = append(ops, op)
case has("use"):
name, ok := v.(string)
if !ok {
return nil, fmt.Errorf(`expected string but got "%v" of type %T`, v, v)
}
op, err := operation.Build(name, opConfig)
if err != nil {
return nil, err
}
ops = append(ops, op)
default:
return nil, fmt.Errorf(`pipeline entry is missing one of "use", "source" or "target", got %#v`, opConfig)
}
}
return ops, nil
}
func Process(contents []cabret.Content, ops []cabret.Operation) ([]cabret.Content, error) {
for _, op := range ops {
var err error
contents, err = cabret.ProcessOperation(op, contents)
if err != nil {
return nil, err
}
}
return contents, nil
}
Loading…
Cancel
Save