gristlabs_grist-core/sandbox/pyodide/packages.js
Alex Hall d72f177be0
Upgrade to Pyodide 0.23.4 (with Python 3.11) and compile dependencies (#603)
* Build packages with latest pyodide, compile wheels to use pyc files, output package_filenames.json
2023-08-02 20:15:53 +02:00

120 lines
3.8 KiB
JavaScript

const path = require('path');
const fs = require('fs');
const fetch = require('node-fetch');
async function listLibs(src) {
const txt = fs.readFileSync(path.join(__dirname, '..', 'requirements3.txt'), 'utf8');
const libs = {};
for (const line of txt.split(/\r?\n/)) {
const raw = line.split('#')[0];
if (!raw.includes('==')) { continue; }
const [name, version] = line.split('==');
libs[name] = version;
}
const hits = [];
const misses = [];
const toLoad = [];
const material = fs.readdirSync(src);
for (const [lib, version] of Object.entries(libs)) {
const nlib = lib.replace(/-/g, '_');
const info = {
name: lib,
standardName: nlib,
version: version,
}
try {
const found = material.filter(m => m.startsWith(`${nlib}-${version}-`));
if (found.length !== 1) {
throw new Error('did not find 1');
}
const fname = found[0];
info.fullName = path.join(src, fname);
info.fileName = fname;
toLoad.push(info);
hits.push(lib);
} catch (e) {
misses.push(info);
}
}
return {
available: toLoad,
misses,
};
}
exports.listLibs = listLibs;
async function findOnDisk(src, dest) {
console.log(`Organizing packages on disk`, {src, dest});
fs.mkdirSync(dest, {recursive: true});
let libs = (await listLibs(src));
for (const lib of libs.available) {
fs.copyFileSync(lib.fullName, path.join(dest, lib.fileName));
fs.writeFileSync(path.join(dest, `${lib.name}-${lib.version}.json`),
JSON.stringify({
name: lib.name,
version: lib.version,
fileName: lib.fileName,
}, null, 2));
console.log("Copied", {
content: path.join(dest, lib.fileName),
meta: path.join(dest, `${lib.name}-${lib.version}.json`),
});
}
libs = await listLibs(dest);
fs.writeFileSync(path.join(__dirname, `package_filenames.json`),
JSON.stringify(libs.available.map(lib => lib.fileName), null, 2));
console.log(`Cached`, {libs: libs.available.map(lib => lib.name)});
console.log(`Missing`, {libs: libs.misses.map(lib => lib.name)});
}
async function findOnNet(src, dest) {
console.log(`Caching packages on disk`, {src, dest});
fs.mkdirSync(dest, {recursive: true});
let libs = await listLibs(dest);
console.log(`Cached`, {libs: libs.available.map(lib => lib.name)});
for (const lib of libs.misses) {
console.log('Fetching', lib);
const url = new URL(src);
url.pathname = url.pathname + lib.name + '-' + lib.version + '.json';
const result = await fetch(url.href);
if (result.status === 200) {
const data = await result.json();
const url2 = new URL(src);
url2.pathname = url2.pathname + data.fileName;
const result2 = await fetch(url2.href);
if (result2.status === 200) {
fs.writeFileSync(path.join(dest, `${lib.name}-${lib.version}.json`),
JSON.stringify(data, null, 2));
fs.writeFileSync(path.join(dest, data.fileName),
await result2.buffer());
} else {
console.error("No payload available", {lib});
}
} else {
console.error("No metadata available", {lib});
}
}
libs = await listLibs(dest);
console.log(`Missing`, {libs: libs.misses.map(lib => lib.name)});
}
async function main(src, dest) {
if (!src) {
console.error('please supply a source');
process.exit(1);
}
if (!dest) {
console.error('please supply a destination');
process.exit(1);
}
if (src.startsWith('http:') || src.startsWith('https:')) {
await findOnNet(src, dest);
return;
}
await findOnDisk(src, dest);
}
if (require.main === module) {
main(...process.argv.slice(2)).catch(e => console.error(e));
}