aziis98/cabret
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rewrite to new yaml format

Browse Source
This commit is contained in:
aziis98
2022-12-31 15:31:07 +01:00
parent 9ebc4715ed
commit 46715cf033
21 changed files with 462 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+3
View File
@@ -1,4 +1,7 @@
.env
*.local*
bin/
dist/
.vscode/
cabret.go
+36 -13
View File
@@ -1,6 +1,20 @@
package cabret
const MatchResult = "MatchResult"
import (
"fmt"
"log"
)
func init() {
log.SetFlags(0)
}
const MatchResultKey = "MatchResult"
type MatchResult struct {
File string
Captures map[string]string
}
type Map map[string]any
@@ -20,21 +34,29 @@ type Content struct {
Metadata Map
}
type Operation interface {
Load(config map[string]any) error
}
type ListOperation interface {
MapAll(contents []Content) ([]Content, error)
Operation
ProcessList(contents []Content) ([]Content, error)
}
type ItemOperation interface {
FlatMap(content Content) (*Content, error)
Operation
ProcessItem(content Content) (*Content, error)
}
type FlatMapToMapAll struct{ FlatMapOperation }
func ProcessOperation(op Operation, inputs []Content) ([]Content, error) {
switch op := op.(type) {
case ListOperation:
return op.ProcessList(inputs)
func (op FlatMapToMapAll) MapAll(contents []Content) ([]Content, error) {
mapped := []Content{}
for _, item := range contents {
result, err := op.FlatMap(item)
case ItemOperation:
outputs := []Content{}
for _, item := range inputs {
result, err := op.ProcessItem(item)
if err != nil {
return nil, err
}
@@ -43,9 +65,10 @@ func (op FlatMapToMapAll) MapAll(contents []Content) ([]Content, error) {
if result == nil {
continue
}
mapped = append(mapped, *result)
outputs = append(outputs, *result)
}
return outputs, nil
default:
return nil, fmt.Errorf(`invalid operation type %T`, op)
}
return mapped, nil
}
cmd/cabret/main.go
+3 -5
View File
@@ -9,17 +9,15 @@ import (
func main() {
log.SetFlags(0)
log.Printf("Rendering current project")
site, err := config.ReadCabretfile("./Cabretfile.yaml")
cabretfile, err := config.ReadCabretfile("./Cabretfile.yaml")
if err != nil {
log.Fatal(err)
}
// repr.Println(site)
// repr.Println(cabretfile)
if err := exec.Execute(site); err != nil {
if err := exec.Execute(cabretfile); err != nil {
log.Fatal(err)
}
}
config/config.go
+7 -10
View File
@@ -6,24 +6,21 @@ import (
"gopkg.in/yaml.v3"
)
// Operation is an enum of various operations
// Operation should have at least one key in "source", "use", "target". The remaining keys are options for that operation
type Operation map[string]any
type EntryPoint struct {
Source string `yaml:",omitempty"`
Pipeline []Operation `yaml:",omitempty"`
type Pipeline struct {
Pipeline []Operation `yaml:"pipeline"`
}
type Options struct {
type BuildOptions struct {
// Excludes lists files and folders to globally exclude from compilation
Excludes []string `yaml:",omitempty"`
// Include []string `yaml:",omitempty"`
Output string `yaml:",omitempty"`
}
// Cabretfile has some configuration for the
type Cabretfile struct {
Options Options `yaml:",omitempty"`
EntryPoints []*EntryPoint `yaml:"entryPoints"`
Options BuildOptions
Build []Pipeline
}
func ReadCabretfile(file string) (*Cabretfile, error) {
examples/basic/Cabretfile.yaml
+8 -6
View File
@@ -1,12 +1,14 @@
entryPoints:
build:
- pipeline:
- source: index.html
pipeline:
- layout: layouts/base.html
- use: layout
path: layouts/base.html
- target: dist/index.html
- pipeline:
- source: posts/{id}.md
pipeline:
- plugin: markdown
- layout: layouts/base.html
- use: markdown
- use: layout
path: layouts/base.html
- target: dist/posts/{id}/index.html
examples/basic/index.html
+2 -1
View File
@@ -1,5 +1,6 @@
<h1>My Website</h1>
<p>
Lorem ipsum dolor sit amet consectetur adipisicing elit. Minima, eveniet, dolorum amet, cupiditate quae excepturi aspernatur dolor voluptatem obcaecati ratione quas? Et explicabo illum iure eius porro, dolor quos doloremque!
Lorem ipsum dolor sit amet consectetur adipisicing elit. Minima, eveniet, dolorum amet, cupiditate quae excepturi aspernatur dolor
voluptatem obcaecati ratione quas? Et explicabo illum iure eius porro, dolor quos doloremque!
</p>
exec/exec.go
+4 -107
View File
@@ -1,124 +1,21 @@
package exec
import (
"log"
"mime"
"os"
gopath "path"
"github.com/alecthomas/repr"
"github.com/aziis98/cabret"
"github.com/aziis98/cabret/config"
"github.com/aziis98/cabret/operation"
"github.com/aziis98/cabret/path"
"github.com/aziis98/cabret/pipeline"
)
type matchResult struct {
file string
context map[string]string
}
func BuildOperation(op config.Operation) cabret.Operation {
if path, ok := op["read"]; ok {
return cabret.FlatMapToMapAll{&operation.Target{
PathTemplate: path.(string),
}}
}
if path, ok := op["write"]; ok {
return cabret.FlatMapToMapAll{&operation.Target{
PathTemplate: path.(string),
}}
}
if name, ok := op["plugin"]; ok {
switch name {
case "layout":
path := op["path"].(string)
delete(op, "path")
return cabret.FlatMapToMapAll{&operation.Layout{
TemplateFilesPattern: path,
Options: op,
}}
case "markdown":
return cabret.FlatMapToMapAll{&operation.Markdown{
Options: op,
}}
default:
log.Fatalf(`invalid operation: %s`, name)
}
}
return nil
}
func Execute(cfg *config.Cabretfile) error {
files, err := cabret.FindFiles([]string{})
for _, p := range cfg.Build {
ops, err := pipeline.Parse(p)
if err != nil {
return err
}
// the first index is the entrypoint ID, the second is for the array of matched files for this entrypoint
entryPointMatches := make([][]matchResult, len(cfg.EntryPoints))
// load all files to process
for id, ep := range cfg.EntryPoints {
pat, err := path.ParsePattern(ep.Source)
if err != nil {
if _, err := pipeline.Process([]cabret.Content{}, ops); err != nil {
return err
}
matchedFiles := []matchResult{}
for _, f := range files {
if ok, ctx, _ := pat.Match(f); ok {
log.Printf(`[Preload] [EntryPoint %d] Found "%s": %#v`, id, f, ctx)
matchedFiles = append(matchedFiles, matchResult{f, ctx})
}
}
entryPointMatches[id] = matchedFiles
}
// TODO: preload all metadata...
// process all entrypoints
for id, ep := range cfg.EntryPoints {
log.Printf(`[EntryPoint %d] starting to process %d file(s)`, id, len(entryPointMatches[id]))
for _, m := range entryPointMatches[id] {
log.Printf(`[EntryPoint %d] ["%s"] reading file`, id, m.file)
data, err := os.ReadFile(m.file)
if err != nil {
return err
}
content := cabret.Content{
Type: mime.TypeByExtension(gopath.Ext(m.file)),
Data: data,
Metadata: cabret.Map{
cabret.MatchResult: m.context,
},
}
for i, opConfig := range ep.Pipeline {
op := BuildOperation(opConfig)
log.Printf(`[EntryPoint %d] ["%s"] [Operation(%d)] applying %s`, id, m.file, i, repr.String(op))
newContent, err := op.Process(content)
if err != nil {
return err
}
if newContent == nil {
break
}
// log.Printf(`[EntryPoint %d] ["%s"] [Operation(%d)] [Metadata] %s`, id, m.file, i, repr.String(newContent.Metadata, repr.Indent(" ")))
content = *newContent
}
log.Printf(`[EntryPoint %d] ["%s"] done`, id, m.file)
}
}
return nil
operation/categorize.go
+39
View File
@@ -0,0 +1,39 @@
package operation
import (
"fmt"
"github.com/aziis98/cabret"
)
func init() {
registerType("categorize", &Categorize{})
}
type Categorize struct {
Key string
// Operation to be executed for each category
Operation cabret.Operation
}
func (op *Categorize) Load(config map[string]any) error {
{
v, ok := config["key"]
if !ok {
return fmt.Errorf(`missing "key" field`)
}
key, ok := v.(string)
if !ok {
return fmt.Errorf(`expected string but got "%v" of type %T`, v, v)
}
op.Key = key
}
return nil
}
func (op *Categorize) Process(content cabret.Content) (*cabret.Content, error) {
return nil, nil
}
operation/group.go
-12
View File
@@ -1,12 +0,0 @@
package operation
import "github.com/aziis98/cabret"
type GroupBy struct {
Key string
}
func (op GroupBy) Process(content cabret.Content) (*cabret.Content, error) {
return nil, nil
}
operation/layout.go
+57 -28
View File
@@ -1,38 +1,73 @@
package operation
import (
"html/template"
"fmt"
goHtmlTemplate "html/template"
"log"
"mime"
"path/filepath"
"strings"
gopath "path"
"github.com/alecthomas/repr"
"github.com/aziis98/cabret"
"github.com/aziis98/cabret/operation/layout"
"github.com/aziis98/cabret/operation/template"
"github.com/aziis98/cabret/util"
)
var HtmlMimeType = mime.TypeByExtension(".html")
var _ cabret.FlatMapOperation = Layout{}
type Layout struct {
// TemplateFilesPattern is a comma separated list of unix glob patterns
TemplateFilesPattern string
Options map[string]any
func init() {
registerType("layout", &Layout{})
}
func (op Layout) FlatMap(content cabret.Content) (*cabret.Content, error) {
var tmpl layout.Template
type Layout struct {
// TemplatePatterns is a list of glob patterns of templates that will be loaded
TemplatePatterns []string
}
patterns := strings.Split(op.TemplateFilesPattern, ",")
func (op *Layout) Load(config map[string]any) error {
if v, ok := config[ShortFormValueKey]; ok {
globPatternsStr, ok := v.(string)
if !ok {
return fmt.Errorf(`expected a comma separated list of glob patterns but got "%v" of type %T`, v, v)
}
globPatterns := strings.Split(globPatternsStr, ",")
for _, pat := range globPatterns {
op.TemplatePatterns = append(op.TemplatePatterns, strings.TrimSpace(pat))
}
return nil
}
if v, ok := config["paths"]; ok {
globPatterns, ok := v.([]string)
if !ok {
return fmt.Errorf(`expected a list of glob patterns but got "%v" of type %T`, v, v)
}
for _, pat := range globPatterns {
op.TemplatePatterns = append(op.TemplatePatterns, strings.TrimSpace(pat))
}
return nil
}
if v, ok := config["path"]; ok {
globPatternStr, ok := v.(string)
if !ok {
return fmt.Errorf(`expected a glob pattern but got "%v" of type %T`, v, v)
}
op.TemplatePatterns = []string{strings.TrimSpace(globPatternStr)}
return nil
}
return fmt.Errorf(`invalid config`)
}
func (op Layout) ProcessItem(content cabret.Content) (*cabret.Content, error) {
// expand glob patterns
tmplFiles := []string{}
for _, pat := range patterns {
for _, pat := range op.TemplatePatterns {
files, err := filepath.Glob(strings.TrimSpace(pat))
if err != nil {
return nil, err
@@ -41,28 +76,22 @@ func (op Layout) FlatMap(content cabret.Content) (*cabret.Content, error) {
tmplFiles = append(tmplFiles, files...)
}
log.Printf(`[Layout] template pattern "%s" expanded to %s`, op.TemplateFilesPattern, repr.String(tmplFiles))
if gopath.Ext(tmplFiles[0]) == ".html" {
var err error
if tmpl, err = layout.NewHtmlTemplate(tmplFiles...); err != nil {
// create template
tmpl, err := template.ParseFiles(tmplFiles...)
if err != nil {
return nil, err
}
} else {
var err error
if tmpl, err = layout.NewTextTemplate(tmplFiles...); err != nil {
return nil, err
}
}
ctx := util.CloneMap(content.Metadata)
if content.Type == HtmlMimeType {
ctx["Content"] = template.HTML(content.Data)
ctx["Content"] = goHtmlTemplate.HTML(content.Data)
} else {
ctx["Content"] = content.Data
}
log.Printf(`[operation.Layout] rendering into layout "%s"`, strings.Join(op.TemplatePatterns, ", "))
data, err := tmpl.Render(ctx)
if err != nil {
return nil, err
operation/layout/layout.go
-5
View File
@@ -1,5 +0,0 @@
package layout
type Template interface {
Render(ctx map[string]any) ([]byte, error)
}
operation/markdown.go
+11 -2
View File
@@ -2,6 +2,7 @@ package operation
import (
"bytes"
"log"
"github.com/aziis98/cabret"
"github.com/iancoleman/strcase"
@@ -11,13 +12,19 @@ import (
"github.com/yuin/goldmark/parser"
)
var _ cabret.FlatMapOperation = Markdown{}
func init() {
registerType("markdown", &Markdown{})
}
type Markdown struct {
Options map[string]any
}
func (op Markdown) FlatMap(content cabret.Content) (*cabret.Content, error) {
func (op *Markdown) Load(config map[string]any) error {
return nil
}
func (op Markdown) ProcessItem(content cabret.Content) (*cabret.Content, error) {
md := goldmark.New(
goldmark.WithExtensions(
extension.GFM,
@@ -30,6 +37,8 @@ func (op Markdown) FlatMap(content cabret.Content) (*cabret.Content, error) {
var buf bytes.Buffer
log.Printf(`[operation.Markdown] rendering markdown`)
context := parser.NewContext()
if err := md.Convert(content.Data, &buf, parser.WithContext(context)); err != nil {
panic(err)
operation/read.go
-40
View File
@@ -1,40 +0,0 @@
package operation
import (
"mime"
"os"
gopath "path"
"path/filepath"
"github.com/aziis98/cabret"
)
var _ cabret.ListOperation = Read{}
type Read struct {
Patterns []string
}
func (op Read) MapAll(contents []cabret.Content) ([]cabret.Content, error) {
for _, pattern := range op.Patterns {
files, err := filepath.Glob(pattern)
if err != nil {
return nil, err
}
for _, file := range files {
data, err := os.ReadFile(file)
if err != nil {
return nil, err
}
contents = append(contents, cabret.Content{
Type: mime.TypeByExtension(gopath.Ext(file)),
Data: data,
Metadata: cabret.Map{},
})
}
}
return contents, nil
}
operation/registry.go
+34
View File
@@ -0,0 +1,34 @@
package operation
import (
"fmt"
"log"
"reflect"
"github.com/aziis98/cabret"
)
// ShortFormValueKey is used by some operations that support an inline form
const ShortFormValueKey = "value"
var registry = map[string]reflect.Type{}
func registerType(name string, op cabret.Operation) {
typ := reflect.TypeOf(op).Elem()
log.Printf(`[operation] registered type "%v"`, typ)
registry[name] = typ
}
func Build(name string, options map[string]any) (cabret.Operation, error) {
typ, ok := registry[name]
if !ok {
return nil, fmt.Errorf(`no registered operation named %q`, name)
}
op := reflect.New(typ).Interface().(cabret.Operation)
if err := op.Load(options); err != nil {
return nil, err
}
return op, nil
}
operation/registry_test.go
+18
View File
@@ -0,0 +1,18 @@
package operation
import (
"testing"
"gotest.tools/assert"
)
func TestBuild(t *testing.T) {
op, err := Build("categorize", map[string]any{
"key": "tags",
})
assert.NilError(t, err)
assert.DeepEqual(t, op, &Categorize{
Key: "tags",
})
}
operation/source.go
+88
View File
@@ -0,0 +1,88 @@
package operation
import (
"fmt"
"log"
"mime"
"os"
gopath "path"
"github.com/aziis98/cabret"
"github.com/aziis98/cabret/path"
)
func init() {
registerType("source", &Source{})
}
// Source is a ListOperation that appends the matched files to the processing items
type Source struct {
Patterns []string
}
func (op *Source) Load(config map[string]any) error {
if v, ok := config[ShortFormValueKey]; ok {
pattern, ok := v.(string)
if !ok {
return fmt.Errorf(`expected pattern but got "%v" of type %T`, v, v)
}
op.Patterns = []string{pattern}
return nil
}
if v, ok := config["paths"]; ok {
patterns, ok := v.([]string)
if !ok {
return fmt.Errorf(`expected list of patterns but got "%v" of type %T`, v, v)
}
op.Patterns = patterns
return nil
}
return fmt.Errorf(`invalid config for "source": %#v`, config)
}
func (op Source) ProcessList(contents []cabret.Content) ([]cabret.Content, error) {
files, err := cabret.FindFiles([]string{})
if err != nil {
return nil, err
}
matches := []cabret.MatchResult{}
for _, patternStr := range op.Patterns {
pat, err := path.ParsePattern(patternStr)
if err != nil {
return nil, err
}
for _, f := range files {
if ok, captures, _ := pat.Match(f); ok {
matches = append(matches, cabret.MatchResult{
File: f,
Captures: captures,
})
}
}
}
for _, m := range matches {
log.Printf(`[operation.Source] reading "%s"`, m.File)
data, err := os.ReadFile(m.File)
if err != nil {
return nil, err
}
contents = append(contents, cabret.Content{
Type: mime.TypeByExtension(gopath.Ext(m.File)),
Data: data,
Metadata: cabret.Map{
cabret.MatchResultKey: m.Captures,
},
})
}
return contents, nil
}
operation/target.go
+23 -4
View File
@@ -2,6 +2,7 @@ package operation
import (
"fmt"
"log"
"os"
gopath "path"
@@ -10,20 +11,38 @@ import (
"github.com/aziis98/cabret/path"
)
var _ cabret.FlatMapOperation = Target{}
func init() {
registerType("target", &Target{})
}
type Target struct {
PathTemplate string
}
func (op Target) FlatMap(c cabret.Content) (*cabret.Content, error) {
mr, ok := c.Metadata[cabret.MatchResult].(map[string]string)
func (op *Target) Load(config map[string]any) error {
if v, ok := config[ShortFormValueKey]; ok {
template, ok := v.(string)
if !ok {
return nil, fmt.Errorf(`invalid match result type %T`, c.Metadata[cabret.MatchResult])
return fmt.Errorf(`expected pattern but got "%v" of type %T`, v, v)
}
op.PathTemplate = template
return nil
}
return fmt.Errorf(`invalid config for "target": %#v`, config)
}
func (op Target) ProcessItem(c cabret.Content) (*cabret.Content, error) {
mr, ok := c.Metadata[cabret.MatchResultKey].(map[string]string)
if !ok {
return nil, fmt.Errorf(`invalid match result type %T`, c.Metadata[cabret.MatchResultKey])
}
target := path.RenderTemplate(op.PathTemplate, mr)
log.Printf(`[operation.Target] writing "%s"`, target)
if err := os.MkdirAll(gopath.Dir(target), 0777); err != nil {
return nil, err
}
operation/layout/html.go → operation/template/html.go
+7 -8
View File
@@ -1,18 +1,18 @@
package layout
package template
import (
"bytes"
"html/template"
goHtmlTemplate "html/template"
)
var _ Template = HtmlTemplate{}
type HtmlTemplate struct {
*template.Template
*goHtmlTemplate.Template
}
func NewHtmlTemplate(files ...string) (*HtmlTemplate, error) {
t, err := template.ParseFiles(files...)
t, err := goHtmlTemplate.ParseFiles(files...)
if err != nil {
return nil, err
}
@@ -21,11 +21,10 @@ func NewHtmlTemplate(files ...string) (*HtmlTemplate, error) {
}
func (t HtmlTemplate) Render(ctx map[string]any) ([]byte, error) {
var b bytes.Buffer
if err := t.Template.Execute(&b, ctx); err != nil {
var buf bytes.Buffer
if err := t.Template.Execute(&buf, ctx); err != nil {
return nil, err
}
return b.Bytes(), nil
return buf.Bytes(), nil
}
operation/template/template.go
+17
View File
@@ -0,0 +1,17 @@
package template
import (
"path/filepath"
)
type Template interface {
Render(ctx map[string]any) ([]byte, error)
}
func ParseFiles(files ...string) (Template, error) {
if filepath.Ext(files[0]) == ".html" {
return NewHtmlTemplate(files...)
}
return NewTextTemplate(files...)
}
operation/layout/text.go → operation/template/text.go
+7 -8
View File
@@ -1,18 +1,18 @@
package layout
package template
import (
"bytes"
"text/template"
goTextTemplate "text/template"
)
var _ Template = TextTemplate{}
type TextTemplate struct {
*template.Template
*goTextTemplate.Template
}
func NewTextTemplate(files ...string) (*TextTemplate, error) {
t, err := template.ParseFiles(files...)
t, err := goTextTemplate.ParseFiles(files...)
if err != nil {
return nil, err
}
@@ -21,11 +21,10 @@ func NewTextTemplate(files ...string) (*TextTemplate, error) {
}
func (t TextTemplate) Render(ctx map[string]any) ([]byte, error) {
var b bytes.Buffer
if err := t.Template.Execute(&b, ctx); err != nil {
var buf bytes.Buffer
if err := t.Template.Execute(&buf, ctx); err != nil {
return nil, err
}
return b.Bytes(), nil
return buf.Bytes(), nil
}
pipeline/pipeline.go
+88
View File
@@ -0,0 +1,88 @@
package pipeline
import (
"fmt"
"github.com/aziis98/cabret"
"github.com/aziis98/cabret/config"
"github.com/aziis98/cabret/operation"
)
func switchMap(m map[string]any, v *any) func(k string) bool {
return func(k string) bool {
val, ok := m[k]
if ok {
*v = val
}
return ok
}
}
func Parse(p config.Pipeline) ([]cabret.Operation, error) {
ops := []cabret.Operation{}
for _, opConfig := range p.Pipeline {
var v any
has := switchMap(opConfig, &v)
switch {
case has("source"):
value, ok := v.(string)
if !ok {
return nil, fmt.Errorf(`expected string but got "%v" of type %T`, v, v)
}
opConfig[operation.ShortFormValueKey] = value
op := &operation.Source{}
if err := op.Load(opConfig); err != nil {
return nil, err
}
ops = append(ops, op)
case has("target"):
value, ok := v.(string)
if !ok {
return nil, fmt.Errorf(`expected string but got "%v" of type %T`, v, v)
}
opConfig[operation.ShortFormValueKey] = value
op := &operation.Target{}
if err := op.Load(opConfig); err != nil {
return nil, err
}
ops = append(ops, op)
case has("use"):
name, ok := v.(string)
if !ok {
return nil, fmt.Errorf(`expected string but got "%v" of type %T`, v, v)
}
op, err := operation.Build(name, opConfig)
if err != nil {
return nil, err
}
ops = append(ops, op)
default:
return nil, fmt.Errorf(`pipeline entry is missing one of "use", "source" or "target", got %#v`, opConfig)
}
}
return ops, nil
}
func Process(contents []cabret.Content, ops []cabret.Operation) ([]cabret.Content, error) {
for _, op := range ops {
var err error
contents, err = cabret.ProcessOperation(op, contents)
if err != nil {
return nil, err
}
}
return contents, nil
}