From 915bd5ee65854fba98b47429139338047ebc8e06 Mon Sep 17 00:00:00 2001 From: Antonio De Lucreziis Date: Thu, 30 Jun 2022 04:26:33 +0200 Subject: [PATCH] Prototype of DirStore and some routes for working with buckets --- config.go | 4 ++ database/database.go | 131 ++++++++++++++++++++++++++++++++++++++- database/mem.go | 7 --- database/sqlite.go | 19 ------ go.mod | 2 +- main.go | 9 ++- routes/api.go | 87 +++++++++++++++++++++++++- routes/router.go | 2 +- store/store.go | 144 +++++++++++++++++++++++++++++++++++++++++++ utils/util.go | 23 +++++++ 10 files changed, 393 insertions(+), 35 deletions(-) delete mode 100644 database/mem.go delete mode 100644 database/sqlite.go create mode 100644 store/store.go create mode 100644 utils/util.go diff --git a/config.go b/config.go index fd1916f..7b103f8 100644 --- a/config.go +++ b/config.go @@ -11,6 +11,8 @@ var Config struct { Mode string Host string BaseURL string + + AdminPassword string } func loadEnv(key string, defaultValue ...string) string { @@ -34,4 +36,6 @@ func init() { Config.Mode = loadEnv(os.Getenv("MODE"), "development") Config.Host = loadEnv(os.Getenv("HOST"), ":4000") Config.BaseURL = loadEnv(os.Getenv("HOST"), "http://localhost:4000") + + Config.AdminPassword = loadEnv(os.Getenv("ADMIN_PASSWORD"), "secret") } diff --git a/database/database.go b/database/database.go index 314c3b1..d1f9911 100644 --- a/database/database.go +++ b/database/database.go @@ -1,3 +1,132 @@ package database -type Database interface{} +import ( + "encoding/json" + "log" + "os" + + "git.phc.dm.unipi.it/phc/storage/store" +) + +type Database interface { + Buckets() ([]string, error) + CreateBucket(bucket string, options ...any) error + + Bucket(bucket string) (store.Store, error) +} + +type bucketInfo struct { + Name string `json:"name"` + Path string `json:"path"` +} + +type jsonDB struct { + file string + + BucketsInfo map[string]bucketInfo `json:"buckets"` +} + +func NewJSON(file string) Database { + db := &jsonDB{ + file: file, + BucketsInfo: map[string]bucketInfo{}, + } + + err := db.load() + if err != nil { + panic(err) + } + + return db +} + +func (db *jsonDB) setup() error { + if _, err := os.Stat(db.file); !os.IsNotExist(err) { + return nil + } + + log.Printf("missing %q, creating empty database file", db.file) + + f, err := os.Create(db.file) + defer f.Close() + if err != nil { + return err + } + + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + if err := enc.Encode(&db); err != nil { + return err + } + + return nil +} + +func (db *jsonDB) store() error { + f, err := os.Create(db.file) + defer f.Close() + if err != nil { + return err + } + + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + if err := enc.Encode(&db); err != nil { + return err + } + + return nil +} + +func (db *jsonDB) load() error { + if err := db.setup(); err != nil { + return err + } + + f, err := os.Open(db.file) + defer f.Close() + if err != nil { + return err + } + + if err := json.NewDecoder(f).Decode(&db); err != nil { + return err + } + + return nil +} + +func (db *jsonDB) Buckets() ([]string, error) { + db.load() + + buckets := make([]string, 0, len(db.BucketsInfo)) + + for _, b := range db.BucketsInfo { + buckets = append(buckets, b.Name) + } + + return buckets, nil +} + +func (db *jsonDB) CreateBucket(bucket string, options ...any) error { + db.load() + defer db.store() + + bi := bucketInfo{Name: bucket, Path: bucket + "/"} + if len(options) > 0 { + bi.Path = options[0].(string) + } + + db.BucketsInfo[bucket] = bi + + return nil +} + +func (db *jsonDB) Bucket(bucket string) (store.Store, error) { + b := db.BucketsInfo[bucket] + + return &store.DirStore{ + BaseDir: b.Path, + Prefix: 2, + }, nil +} diff --git a/database/mem.go b/database/mem.go deleted file mode 100644 index 7fd6e1e..0000000 --- a/database/mem.go +++ /dev/null @@ -1,7 +0,0 @@ -package database - -type memDB struct {} - -func NewInMemoryDB() Database { - return &memDB{} -} \ No newline at end of file diff --git a/database/sqlite.go b/database/sqlite.go deleted file mode 100644 index 4b52bb0..0000000 --- a/database/sqlite.go +++ /dev/null @@ -1,19 +0,0 @@ -package database - -import "database/sql" - -// Uncomment if actually using SQLite, the first build will be fairly slow -// import _ "github.com/mattn/go-sqlite3" - -type sqliteDatabase struct { - Db *sql.DB -} - -func NewSQLite(filename string) (Database, error) { - db, err := sql.Open("sqlite3", filename) - if err != nil { - return nil, err - } - - return &sqliteDatabase{db}, nil -} diff --git a/go.mod b/go.mod index f81d3fe..a7ba831 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module server +module git.phc.dm.unipi.it/phc/storage go 1.18 diff --git a/main.go b/main.go index 37f1dd7..e351dc7 100644 --- a/main.go +++ b/main.go @@ -4,20 +4,19 @@ import ( "bufio" "log" "os/exec" - "server/database" - "server/routes" "strings" + "git.phc.dm.unipi.it/phc/storage/database" + "git.phc.dm.unipi.it/phc/storage/routes" + "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/logger" "github.com/gofiber/fiber/v2/middleware/recover" ) func main() { - db := database.NewInMemoryDB() - router := &routes.Router{ - Database: db, + Database: database.NewJSON("database.local.json"), } app := fiber.New() diff --git a/routes/api.go b/routes/api.go index 0db67a0..3872b3c 100644 --- a/routes/api.go +++ b/routes/api.go @@ -1,9 +1,94 @@ package routes -import "github.com/gofiber/fiber/v2" +import ( + "bytes" + "log" + + "github.com/gofiber/fiber/v2" +) func (r *Router) Api(api fiber.Router) { api.Get("/status", func(c *fiber.Ctx) error { return c.JSON("ok") }) + + api.Get("/buckets", func(c *fiber.Ctx) error { + buckets, err := r.Database.Buckets() + if err != nil { + return err + } + + return c.JSON(buckets) + }) + + api.Post("/buckets", func(c *fiber.Ctx) error { + var req struct { + Bucket string `json:"bucket"` + Path string `json:"path"` + } + + log.Printf("%v", string(c.Body())) + + if err := c.BodyParser(&req); err != nil { + return err + } + + opts := []any{} + if req.Path != "" { + opts = append(opts, req.Path) + } + + if err := r.Database.CreateBucket(req.Bucket, opts...); err != nil { + return err + } + + return c.JSON("ok") + }) + + api.Post("/buckets/:bucket", func(c *fiber.Ctx) error { + bucket := c.Params("bucket") + + b, err := r.Database.Bucket(bucket) + if err != nil { + return err + } + + ff, err := c.FormFile("file") + if err != nil { + return err + } + + mf, err := ff.Open() + if err != nil { + return err + } + + id, err := b.Create(mf) + if err != nil { + return err + } + + return c.JSON(fiber.Map{ + "bucket": bucket, + "id": id, + }) + }) + + api.Get("/buckets/:bucket/:id", func(c *fiber.Ctx) error { + bucket := c.Params("bucket") + id := c.Params("id") + + buf := &bytes.Buffer{} + + b, err := r.Database.Bucket(bucket) + if err != nil { + return err + } + + if err := b.Read(id, buf); err != nil { + return err + } + + return c.SendStream(buf) + }) } diff --git a/routes/router.go b/routes/router.go index cc61ba6..1fc5443 100644 --- a/routes/router.go +++ b/routes/router.go @@ -1,6 +1,6 @@ package routes -import "server/database" +import "git.phc.dm.unipi.it/phc/storage/database" type Router struct { Database database.Database diff --git a/store/store.go b/store/store.go new file mode 100644 index 0000000..3fb44f7 --- /dev/null +++ b/store/store.go @@ -0,0 +1,144 @@ +package store + +import ( + "fmt" + "io" + "os" + "path" + + "git.phc.dm.unipi.it/phc/storage/utils" +) + +type Store interface { + // Create a new object by reading from the given io.Reader and returns its new id + Create(r io.Reader) (string, error) + + // Read the object identified by "id" into the given io.Writer + Read(id string, w io.Writer) error + + // Update the object identified by "id" by reading from the given io.Reader + Update(id string, r io.Reader) error + + // Delete the object with the given id + Delete(id string) error +} + +// +// In Memory Byte Store +// + +type memStore struct { + objects map[string][]byte +} + +func NewMemStore() Store { + return &memStore{ + objects: map[string][]byte{}, + } +} + +// Create a new object by reading from the given io.Reader and returns its new id +func (m *memStore) Create(r io.Reader) (string, error) { + id := utils.GenerateRandomString(16) + + data, err := io.ReadAll(r) + if err != nil { + return "", err + } + + m.objects[id] = data + return id, nil +} + +// Read the object identified by "id" into the given io.Writer +func (m *memStore) Read(id string, w io.Writer) error { + data, found := m.objects[id] + if !found { + return fmt.Errorf("object with id %q not found", id) + } + + if _, err := w.Write(data); err != nil { + return err + } + + return nil +} + +// Update the object identified by "id" by reading from the given io.Reader +func (m *memStore) Update(id string, r io.Reader) error { + data, err := io.ReadAll(r) + if err != nil { + return err + } + + m.objects[id] = data + return nil +} + +// Delete the object with the given id +func (m *memStore) Delete(id string) error { + if _, found := m.objects[id]; !found { + return fmt.Errorf("object with id %q not found", id) + } + + delete(m.objects, id) + + return nil +} + +// +// Dir Store +// + +type DirStore struct { + // BaseDir is the root folder of this Store + BaseDir string + + // Prefix is the number of letters to use for the first layer of folders + Prefix int +} + +func (d *DirStore) split(id string) (prefix, rest string) { + return id[:d.Prefix], id[d.Prefix:] +} + +func (d *DirStore) Create(r io.Reader) (string, error) { + id := utils.GenerateRandomString(16) + prefix, rest := d.split(id) + + os.MkdirAll(path.Join(d.BaseDir, prefix), os.ModePerm) + + f, err := os.Create(path.Join(d.BaseDir, prefix, rest)) + if err != nil { + return "", err + } + + if _, err := io.Copy(f, r); err != nil { + return "", err + } + + return id, nil +} + +func (d *DirStore) Read(id string, w io.Writer) error { + prefix, rest := d.split(id) + + f, err := os.Open(path.Join(d.BaseDir, prefix, rest)) + if err != nil { + return err + } + + if _, err := io.Copy(w, f); err != nil { + return err + } + + return nil +} + +func (d *DirStore) Update(id string, r io.Reader) error { + panic("TODO: Not implemented") +} + +func (d *DirStore) Delete(id string) error { + panic("TODO: Not implemented") +} diff --git a/utils/util.go b/utils/util.go new file mode 100644 index 0000000..eab4f46 --- /dev/null +++ b/utils/util.go @@ -0,0 +1,23 @@ +package utils + +import ( + "encoding/hex" + "math/rand" + "time" +) + +const alphabet = "1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func GenerateRandomString(n int) string { + b := make([]byte, n/2) + + if _, err := rand.Read(b); err != nil { + panic(err) + } + + return hex.EncodeToString(b) +}