Initial commit

pull/1/head
Antonio De Lucreziis 2 years ago
commit 64702636e6

@ -0,0 +1,6 @@
MODE=development
HOST=:4000
BASE_URL=http://localhost:4000
# For example "node1,node2,node3"
NODES=""

8
.gitignore vendored

@ -0,0 +1,8 @@
.env
*.local*
bin/
.out/
out/
dist/
node_modules/

@ -0,0 +1,69 @@
# Go Vite Kit
Minimal boilerplate project for a Golang server using [Fiber](https://github.com/gofiber/fiber) and [ViteJS](https://github.com/vitejs/vite) for static pages
## Features
- ⚡️ [Go Fiber](https://github.com/gofiber/fiber)
- 📦 [ViteJS](http://vitejs.dev/)
- 🎨 [Sass](https://sass-lang.com/)
- 🗄️ [Sqlite3](https://github.com/mattn/go-sqlite3)
## Architecture
- `frontend/`
This is a Vite project for building all the static pages used by this app.
- `backend/`
This keeps all server related files
- `config/`
Loads env variables and keeps them as globals
- `database/`
Module with a `Database` interface and two implementation: `memDB` is an in-memory database for testing purposes. `sqliteDB` is a wrapper for working with an SQLite database.
- `routes/`
Various functions for configuring all the server routes.
A very important file is `backend/routes/router.go` that contains the `HtmlEntrypoints` variable that is used both by the backend and ViteJS to mount HTML entrypoints.
When building the frontend ViteJS will call `go run ./meta/routes` to read the content of the `HtmlEntrypoints` variable. This is also used while developing to let Vite know add all necessary entrypoints to the dev server.
## Usage
To setup the project first install the required npm packages
```bash
# Install all JS dependencies
$ npm install
```
then you can start versioning the **lock file** of your package manager.
### Development
```bash
# Development
$ MODE=dev go run -v ./cmd/server
# Development with watcher
$ fd -e go | MODE=dev entr -r go run -v ./cmd/server
```
### Production
You can build everything with the following command, it will build first the frontend and then the backend and generate `./out/server`.
```bash
# Build
$ go run -v ./cmd/build
# Run
$ ./out/server
```

@ -0,0 +1,44 @@
package config
import (
"log"
"os"
"strings"
"github.com/joho/godotenv"
)
var (
Mode string
Host string
BaseURL string
// Nodes è una lista di nomi dei vari host
Nodes []string
)
func loadEnv(key string, defaultValue ...string) string {
env := os.Getenv(key)
if len(defaultValue) > 0 && env == "" {
env = defaultValue[0]
}
log.Printf("Environment variable %s = %q", key, env)
return env
}
func init() {
// Setup logger
log.SetFlags(log.Lshortfile | log.Ltime | log.Ldate)
// Load Config
godotenv.Load()
Mode = loadEnv("MODE", "development")
Host = loadEnv("HOST", ":4000")
BaseURL = loadEnv("BASE_URL", "http://localhost:4000")
nodes := loadEnv("NODES", "") // for example "node1,node2,node3"
Nodes = strings.Split(nodes, ",")
}

@ -0,0 +1,72 @@
package database
import (
"time"
"git.phc.dm.unipi.it/phc/cluster-dashboard/backend/executor"
"git.phc.dm.unipi.it/phc/cluster-dashboard/backend/model"
)
// simpleDB è una implementazione di [database.Database] che tiene giusto una cache in memoria e
// quando il server viene riavviato perde tutte le statistiche che ha accumulato. Più avanti si
// potrebbe pensare di scrivere queste informazioni in un file o usare un vero database.
type simpleDB struct {
Executor executor.Service
// lastUpdate tiene traccia di quando abbiamo aggiornato l'ultima volta tutti i dati
lastUpdate *time.Time
// Nodes is a map from hostname to node info
nodes map[string]model.Node
// Jobs is a map from job id to job info
jobs map[int]model.Job
// The following are maps from hostname to a list of sampled temperatures, used memory, used storage space and network upload and download rate.
temperatureSamples map[string][]model.Sample[float64]
memorySamples map[string][]model.Sample[int64]
storageSamples map[string][]model.Sample[int64]
networkUploadSamples map[string][]model.Sample[int64]
networkDownloadSamples map[string][]model.Sample[int64]
}
func NewSimpleDatabase(ex executor.Service) Database {
return &simpleDB{Executor: ex}
}
func (s *simpleDB) GetNodeStatus(hostname string) (*model.Node, error) {
panic("todo")
}
func (s *simpleDB) GetJobStatus(id int) (*model.Job, error) {
panic("todo")
}
func (s *simpleDB) AllNodes() ([]*model.Node, error) {
panic("todo")
}
func (s *simpleDB) AllJobs() ([]*model.Job, error) {
panic("todo")
}
func (s *simpleDB) QueryTemperatureSamples(from, to time.Time) ([]model.Sample[float64], error) {
panic("todo")
}
func (s *simpleDB) QueryMemorySamples(from, to time.Time) ([]model.Sample[int64], error) {
panic("todo")
}
func (s *simpleDB) QueryStorageSamples(from, to time.Time) ([]model.Sample[int64], error) {
panic("todo")
}
func (s *simpleDB) QueryNetworkUploadSamples(from, to time.Time) ([]model.Sample[int64], error) {
panic("todo")
}
func (s *simpleDB) QueryNetworkDownloadSamples(from, to time.Time) ([]model.Sample[int64], error) {
panic("todo")
}

@ -0,0 +1,21 @@
package database
import (
"time"
"git.phc.dm.unipi.it/phc/cluster-dashboard/backend/model"
)
type Database interface {
GetNodeStatus(hostname string) (*model.Node, error)
GetJobStatus(id int) (*model.Job, error)
AllNodes() ([]*model.Node, error)
AllJobs() ([]*model.Job, error)
QueryTemperatureSamples(from, to time.Time) ([]model.Sample[float64], error)
QueryMemorySamples(from, to time.Time) ([]model.Sample[int64], error)
QueryStorageSamples(from, to time.Time) ([]model.Sample[int64], error)
QueryNetworkUploadSamples(from, to time.Time) ([]model.Sample[int64], error)
QueryNetworkDownloadSamples(from, to time.Time) ([]model.Sample[int64], error)
}

@ -0,0 +1,16 @@
package executor
import "time"
// Service is a service that handles executing commands on the main host and does a first processing of the raw data it gets from the system
type Service interface {
SlurmQueue() []string
SlurmJobs() []string
NodeUptime(hostname string) time.Time
Temperature(hostname string) float64
MemoryUsage(hostname string) int64
StorageUsage(hostname string) int64
NetworkUploadDownload(hostname string) (int64, int64)
}

@ -0,0 +1,43 @@
package executor
import "time"
var _ Service = &Mock{}
type Mock struct {
SlurmQueueFunc func() []string
SlurmJobsFunc func() []string
NodeUptimeFunc func(hostname string) time.Time
TemperatureFunc func(hostname string) float64
MemoryUsageFunc func(hostname string) int64
StorageUsageFunc func(hostname string) int64
NetworkUploadDownloadFunc func(hostname string) (int64, int64)
}
func (ex *Mock) SlurmQueue() []string {
return ex.SlurmQueueFunc()
}
func (ex *Mock) SlurmJobs() []string {
return ex.SlurmJobsFunc()
}
func (ex *Mock) NodeUptime(hostname string) time.Time {
return ex.NodeUptimeFunc(hostname)
}
func (ex *Mock) Temperature(hostname string) float64 {
return ex.TemperatureFunc(hostname)
}
func (ex *Mock) MemoryUsage(hostname string) int64 {
return ex.MemoryUsageFunc(hostname)
}
func (ex *Mock) StorageUsage(hostname string) int64 {
return ex.StorageUsageFunc(hostname)
}
func (ex *Mock) NetworkUploadDownload(hostname string) (int64, int64) {
return ex.NetworkUploadDownloadFunc(hostname)
}

@ -0,0 +1,31 @@
package model
import (
"time"
"golang.org/x/exp/constraints"
)
type Numeric = constraints.Ordered
// Job contiene le informazioni su un lavoro di slurm (il nostro gestore di code di lavori)
type Job struct {
Id int `json:"id"`
Name string `json:"name"`
Status string `json:"status"`
Nodes []string `json:"nodes"`
Resources []string `json:"resources"`
}
// Sample di un valore di tipo N (preferibilmente numerico) in un certo momento temporale.
type Sample[N Numeric] struct {
Timestamp time.Time `json:"timestamp"`
Value N `json:"value"`
}
// Node contiene le informazioni relative ad un nodo nel nostro sistema
type Node struct {
Hostname string `json:"hostname"`
StartTime time.Time `json:"startTime"`
}

@ -0,0 +1,9 @@
package routes
import "github.com/gofiber/fiber/v2"
func (r *Router) Api(api fiber.Router) {
api.Get("/status", func(c *fiber.Ctx) error {
return c.JSON("ok")
})
}

@ -0,0 +1,19 @@
package routes
import "git.phc.dm.unipi.it/phc/cluster-dashboard/backend/database"
// Router is the main service that defines all routes, this only depends on the Database service
type Router struct {
Database database.Database
}
type htmlEntrypoint struct {
Route string `json:"route"`
Filename string `json:"filename"`
}
var HtmlEntrypoints = []htmlEntrypoint{
{"/", "./index.html"},
{"/dashboard", "./index.html"},
{"/jobs", "./index.html"},
}

@ -0,0 +1,77 @@
package main
import (
"bufio"
"io"
"log"
"os"
"os/exec"
"path"
"strings"
"github.com/fatih/color"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/logger"
"github.com/gofiber/fiber/v2/middleware/recover"
"git.phc.dm.unipi.it/phc/cluster-dashboard/backend/config"
"git.phc.dm.unipi.it/phc/cluster-dashboard/backend/database"
"git.phc.dm.unipi.it/phc/cluster-dashboard/backend/executor"
"git.phc.dm.unipi.it/phc/cluster-dashboard/backend/routes"
)
const FrontendOutDir = "./out/frontend"
func main() {
ex := &executor.Mock{} // to try out locally this project in a reasonable way
db := database.NewSimpleDatabase(ex)
router := &routes.Router{
Database: db,
}
app := fiber.New()
app.Use(logger.New())
app.Use(recover.New())
app.Static("/", FrontendOutDir)
for _, entrypoint := range routes.HtmlEntrypoints {
app.Get(entrypoint.Route, func(c *fiber.Ctx) error {
return c.SendFile(path.Join(FrontendOutDir, entrypoint.Filename))
})
}
app.Route("/api", router.Api)
if strings.HasPrefix(config.Mode, "dev") {
setupDevServer()
}
log.Fatal(app.Listen(config.Host))
}
func setupDevServer() {
log.Printf(`Running dev server for frontend: "npm run dev"`)
cmd := exec.Command("sh", "-c", "npm run dev")
cmdStdout, _ := cmd.StdoutPipe()
cmdStderr, _ := cmd.StderrPipe()
viteLogger := log.New(os.Stderr, color.HiGreenString("[ViteJS]")+" ", log.Ltime|log.Lmsgprefix)
go func() {
s := bufio.NewScanner(io.MultiReader(cmdStdout, cmdStderr))
for s.Scan() {
viteLogger.Print(s.Text())
}
if err := s.Err(); err != nil {
viteLogger.Fatal(err)
}
}()
err := cmd.Start()
if err != nil {
log.Fatal(err)
}
}

@ -0,0 +1,14 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Homepage</title>
<link rel="stylesheet" href="/styles/main.scss">
</head>
<body>
<script type="module" src="/src/main.jsx"></script>
</body>
</html>

@ -0,0 +1,29 @@
import Router from 'preact-router'
import { route } from 'preact-router'
import { render } from 'preact'
import { useEffect } from 'preact/hooks'
const Redirect = ({ to }) => {
useEffect(() => {
route(to, true)
}, [])
return (
<>
Redirecting to <pre>{to}</pre>...
</>
)
}
const App = () => {
return (
<Router>
<DashboardPage path="/dashboard" />
<JobsPage path="/jobs" />
<Redirect default to="/dashboard" />
</Router>
)
}
render(<App />, document.body)

@ -0,0 +1,43 @@
*,
*::before,
*::after {
box-sizing: border-box;
}
body {
margin: 0;
width: 100%;
min-height: 100vh;
font-family: 'Inter', 'Segoe UI', 'Helvetica', 'Arial', sans-serif;
font-size: 16px;
}
// Headings
$base-font-size: 18px;
$heading-scale: 1.33;
@function pow($number, $exponent) {
$value: 1;
@if $exponent > 0 {
@for $i from 1 through $exponent {
$value: $value * $number;
}
}
@return $value;
}
@for $i from 1 through 5 {
h#{$i} {
margin: 0;
$factor: pow($heading-scale, 5 - $i);
font-size: $base-font-size * $factor;
line-height: 1.5;
}
}

@ -0,0 +1,21 @@
module git.phc.dm.unipi.it/phc/cluster-dashboard
go 1.18
require (
github.com/gofiber/fiber/v2 v2.34.1
github.com/joho/godotenv v1.4.0
)
require (
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/fatih/color v1.15.0 // indirect
github.com/klauspost/compress v1.15.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasthttp v1.37.0 // indirect
github.com/valyala/tcplisten v1.0.0 // indirect
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect
golang.org/x/sys v0.6.0 // indirect
)

@ -0,0 +1,40 @@
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
github.com/gofiber/fiber/v2 v2.34.1 h1:C6saXB7385HvtXX+XMzc5Dqj5S/aEXOfKCW7JNep4rA=
github.com/gofiber/fiber/v2 v2.34.1/go.mod h1:ozRQfS+D7EL1+hMH+gutku0kfx1wLX4hAxDCtDzpj4U=
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U=
github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.37.0 h1:7WHCyI7EAkQMVmrfBhWTCOaeROb1aCBiTopx63LkMbE=
github.com/valyala/fasthttp v1.37.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o=
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

@ -0,0 +1,39 @@
import { join } from 'path'
import express from 'express'
import { createServer as createViteServer } from 'vite'
import { readFile } from 'fs/promises'
import { getBuildRoutes } from './routes.js'
async function createServer() {
const app = express()
// In middleware mode, if you want to use Vite's own HTML serving logic
// use `'html'` as the `middlewareMode` (ref https://vitejs.dev/config/#server-middlewaremode)
const vite = await createViteServer({
server: { middlewareMode: 'html' },
})
const routes = await getBuildRoutes()
console.log(`mounting static routes...`)
for (const [route, file] of Object.entries(routes)) {
const filePath = join('./frontend', file)
console.log(`- "%s" => %s`, route, filePath)
app.get(route, async (req, res) => {
const htmlFile = await readFile(filePath, 'utf8')
const htmlViteHooksFile = await vite.transformIndexHtml(req.originalUrl, htmlFile)
res.setHeader('Content-Type', 'text/html')
return res.send(htmlViteHooksFile)
})
}
app.use(vite.middlewares)
console.log('Started dev server on port :3000')
app.listen(3000)
}
createServer()

@ -0,0 +1,39 @@
import { spawn } from 'child_process'
function transformRoutes(entrypoints) {
return Object.fromEntries(entrypoints.map(({ route, filename }) => [route, filename]))
}
export async function getBuildRoutes() {
// Thanks to ChatGPT
function readCommandOutputAsJSON(command) {
const [cmd, ...args] = command.split(' ')
return new Promise((resolve, reject) => {
const child = spawn(cmd, args)
let stdout = ''
child.stdout.on('data', data => {
stdout += data.toString()
})
child.on('close', code => {
if (code !== 0) {
reject(`Command ${cmd} ${args.join(' ')} failed with code ${code}`)
return
}
try {
const output = JSON.parse(stdout)
resolve(output)
} catch (e) {
reject(`Error parsing JSON output: ${e.message}`)
}
})
})
}
console.log('loading build entrypoints...')
return transformRoutes(await readCommandOutputAsJSON('go run ./meta/routes'))
}

@ -0,0 +1,17 @@
package main
import (
"encoding/json"
"log"
"os"
"git.phc.dm.unipi.it/phc/cluster-dashboard/backend/routes"
)
func main() {
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
if err := enc.Encode(routes.HtmlEntrypoints); err != nil {
log.Fatal(err)
}
}

@ -0,0 +1,23 @@
{
"name": "frontend",
"version": "1.0.0",
"description": "Javascript frontend for this Golang server built using ViteJS",
"main": "index.js",
"type": "module",
"scripts": {
"dev": "node meta/dev-server.js",
"build": "vite build"
},
"license": "MIT",
"devDependencies": {
"@preact/preset-vite": "^2.5.0",
"express": "^4.18.1",
"sass": "^1.53.0",
"vite": "^2.9.13"
},
"dependencies": {
"axios": "^1.4.0",
"preact": "^10.13.2",
"preact-router": "^4.1.0"
}
}

File diff suppressed because it is too large Load Diff

@ -0,0 +1,34 @@
import { defineConfig } from 'vite'
import { getBuildRoutes } from './meta/routes.js'
import preactPlugin from '@preact/preset-vite'
import { join } from 'path'
export default defineConfig(async () => {
const routes = await getBuildRoutes()
console.log('html entrypoints:')
for (const [route, filename] of Object.entries(routes)) {
console.log(`- "${route}" => ${filename}`)
}
console.log()
const entryPoints = Object.values(routes)
return {
root: './frontend',
build: {
outDir: '../out/frontend',
emptyOutDir: true,
rollupOptions: {
input: entryPoints.map(e => join('./frontend', e)),
},
},
server: {
proxy: {
'/api': 'http://127.0.0.1:4000/',
},
},
plugins: [preactPlugin()],
}
})
Loading…
Cancel
Save