From 6a74766690f08e23975607fbcc5dd648737a7f56 Mon Sep 17 00:00:00 2001 From: matlorr Date: Sat, 28 Dec 2024 14:56:29 +0100 Subject: [PATCH 1/4] Added logic to check if artifact to-be-downloaded does not exceed defined disc space threshold --- relay/import.mjs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/relay/import.mjs b/relay/import.mjs index a98ab39..9d34d6d 100644 --- a/relay/import.mjs +++ b/relay/import.mjs @@ -74,9 +74,24 @@ async function doImport (owner, repo, id) { } }) // choose latest artifact + currUsage, threshold = fs.statfs("/", (err, stats) => { + if (err) { + console.error("Failure getting filesystem statistics!") + } + usedBlocks = (stats.blocks - stats.bfree) + usedBytes = usedBlocks * stats.bsize + threshold = stats.blocks * stats.bsize * 0.95 + return (usedBytes, threshold) + }); const artifact = artifacts.data.artifacts .reduce((acc, cur) => acc.created_at < cur.created_at ? cur : acc) artifactId = artifact.id + // Check that size of artifact does not exceed disc space maximum. + artifactSize = artifact.size_in_bytes + if (currUsage + artifactSize >= threshold) { + console.error(`File (${artifactSize}) is exceeding allocated game memory of ${threshold}`) + return + } const url = artifact.archive_download_url // Make sure the download folder exists if (!fs.existsSync(path.join(__dirname, "..", "games"))){ From 39be2aa8e35878d57ad004f7a262a1833196becb Mon Sep 17 00:00:00 2001 From: matlorr Date: Fri, 31 Jan 2025 09:21:56 +0100 Subject: [PATCH 2/4] Update git ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 72e3323..c0120b2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ server/.lake **/.DS_Store logs/ relay/prev_cpu_metric +test.ecosystem.config.cjs From 836bd56e9a7ebb2412cfe1a14f1bc6b6900ebd34 Mon Sep 17 00:00:00 2001 From: matlorr Date: Fri, 31 Jan 2025 16:17:24 +0100 Subject: [PATCH 3/4] Implement function to check if uploaded game will exceed allocated server memory. --- relay/import.mjs | 55 ++++++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/relay/import.mjs b/relay/import.mjs index 9d34d6d..b1f384a 100644 --- a/relay/import.mjs +++ b/relay/import.mjs @@ -1,11 +1,12 @@ import { spawn } from 'child_process' -import fs from 'fs'; +import fs, { stat } from 'fs'; import request from 'request' import requestProgress from 'request-progress' import { Octokit } from 'octokit'; import { fileURLToPath } from 'url'; -import path from 'path'; +import path, { resolve } from 'path'; +import { error } from 'console'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -17,6 +18,7 @@ const octokit = new Octokit({ }) const progress = {} +var exceedingMemoryLimit = false async function runProcess(id, cmd, args, cwd) { return new Promise((resolve, reject) => { @@ -36,6 +38,28 @@ async function runProcess(id, cmd, args, cwd) { }) } + +async function checkAgainstDiscMemory(artifact, maxPercentage) { + return new Promise((resolve, reject) => { + fs.statfs("/", (err, stats) => { + if (err) { + console.log(err); + reject() + } + let artifactBytes = artifact.size_in_bytes; + let totalBytes = stats.blocks * stats.bsize; + let freeBytes = stats.bfree * stats.bsize; + let usedBytes = totalBytes - freeBytes; + let maxUsedBytes = totalBytes * maxPercentage; + if (usedBytes + artifactBytes >= maxUsedBytes) { + exceedingMemoryLimit = true; + } + resolve() + }); + }) +} + + async function download(id, url, dest) { return new Promise((resolve, reject) => { // The options argument is optional so you can omit it @@ -49,7 +73,9 @@ async function download(id, url, dest) { } })) .on('progress', function (state) { - progress[id].output += `Downloaded ${Math.round(state.size.transferred/1024/1024)}MB\n` + console.log('progress', state); + transferredDataSize = Math.round(state.size.transferred/1024/1024) + progress[id].output += `Downloaded ${transferredDataSize}MB\n` }) .on('error', function (err) { reject(err) @@ -73,25 +99,15 @@ async function doImport (owner, repo, id) { 'X-GitHub-Api-Version': '2022-11-28' } }) - // choose latest artifact - currUsage, threshold = fs.statfs("/", (err, stats) => { - if (err) { - console.error("Failure getting filesystem statistics!") - } - usedBlocks = (stats.blocks - stats.bfree) - usedBytes = usedBlocks * stats.bsize - threshold = stats.blocks * stats.bsize * 0.95 - return (usedBytes, threshold) - }); const artifact = artifacts.data.artifacts .reduce((acc, cur) => acc.created_at < cur.created_at ? cur : acc) - artifactId = artifact.id - // Check that size of artifact does not exceed disc space maximum. - artifactSize = artifact.size_in_bytes - if (currUsage + artifactSize >= threshold) { - console.error(`File (${artifactSize}) is exceeding allocated game memory of ${threshold}`) - return + + await checkAgainstDiscMemory(artifact, 0.95); + if (exceedingMemoryLimit === true) { + throw new Error(`Uploading file of size ${Math.round(artifact.size_in_bytes / 1024 / 1024)} (MB) would exceed allocated memory on the server.`); } + + artifactId = artifact.id const url = artifact.archive_download_url // Make sure the download folder exists if (!fs.existsSync(path.join(__dirname, "..", "games"))){ @@ -106,7 +122,6 @@ async function doImport (owner, repo, id) { await runProcess(id, "/bin/bash", [path.join(__dirname, "unpack.sh"), artifactId, owner.toLowerCase(), repo.toLowerCase()], path.join(__dirname, "..")) - // let manifest = fs.readFileSync(`tmp/artifact_${artifactId}_inner/manifest.json`); // manifest = JSON.parse(manifest); // if (manifest.length !== 1) { From 7503739c22bac68540409355bad5bdcc27442f36 Mon Sep 17 00:00:00 2001 From: matlorr Date: Fri, 7 Feb 2025 09:43:35 +0100 Subject: [PATCH 4/4] Add line of contact for upload issues and set reserve disc space as environment variable. --- ecosystem.config.cjs | 2 ++ relay/import.mjs | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ecosystem.config.cjs b/ecosystem.config.cjs index e24dbd5..e0410d1 100644 --- a/ecosystem.config.cjs +++ b/ecosystem.config.cjs @@ -6,6 +6,8 @@ module.exports = { env: { LEAN4GAME_GITHUB_USER: "", LEAN4GAME_GITHUB_TOKEN: "", + RES_DISC_SPACE_PERCENTAGE: 1.0, + ISSUE_CONTACT: "", NODE_ENV: "production", PORT: 8002 }, diff --git a/relay/import.mjs b/relay/import.mjs index b1f384a..deaa98c 100644 --- a/relay/import.mjs +++ b/relay/import.mjs @@ -13,6 +13,8 @@ const __dirname = path.dirname(__filename); const TOKEN = process.env.LEAN4GAME_GITHUB_TOKEN const USERNAME = process.env.LEAN4GAME_GITHUB_USER +const MEM_THRESHOLD = process.env.RES_DISC_SPACE_PERCENTAGE +const CONTACT = process.env.ISSUE_CONTACT const octokit = new Octokit({ auth: TOKEN }) @@ -102,9 +104,11 @@ async function doImport (owner, repo, id) { const artifact = artifacts.data.artifacts .reduce((acc, cur) => acc.created_at < cur.created_at ? cur : acc) - await checkAgainstDiscMemory(artifact, 0.95); + await checkAgainstDiscMemory(artifact, MEM_THRESHOLD); + if (exceedingMemoryLimit === true) { - throw new Error(`Uploading file of size ${Math.round(artifact.size_in_bytes / 1024 / 1024)} (MB) would exceed allocated memory on the server.`); + throw new Error(`Uploading file of size ${Math.round(artifact.size_in_bytes / 1024 / 1024)} (MB) would exceed allocated memory on the server.\n + Please notify server admins via the LEAN zulip instance to resolve this issue.`); } artifactId = artifact.id