Compile libs/input in parallel
This commit is contained in:
parent
7dbc3b0aaa
commit
2798e9adcc
|
|
@ -13,12 +13,19 @@ interface CompilerWithFlags {
|
|||
flags: string[];
|
||||
}
|
||||
|
||||
const STD_C = ['-std=gnu23'];
|
||||
const STD_CPP = ['-std=gnu++26', '-fno-rtti'];
|
||||
|
||||
const WASI_VERSION = '32.0';
|
||||
const WASI_MAJOR = WASI_VERSION.split('.')[0];
|
||||
const wasiArchiveURL = `https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_MAJOR}/wasi-sdk-${WASI_VERSION}-x86_64-linux.tar.gz`;
|
||||
const rtURL = `https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_MAJOR}/libclang_rt-${WASI_VERSION}.tar.gz`;
|
||||
|
||||
const getCompiler = async (): Promise<CompilerWithFlags> => {
|
||||
let compilerPromise: Promise<CompilerWithFlags> | null = null;
|
||||
|
||||
const getCompiler = (): Promise<CompilerWithFlags> => {
|
||||
if (compilerPromise) return compilerPromise;
|
||||
compilerPromise = (async (): Promise<CompilerWithFlags> => {
|
||||
const wasiDir = path.resolve(import.meta.dir, '..', '..', 'dist', 'wasi');
|
||||
const cc: CompilerWithFlags = {
|
||||
cc: 'clang',
|
||||
|
|
@ -47,7 +54,7 @@ const getCompiler = async (): Promise<CompilerWithFlags> => {
|
|||
console.log(`Extracting WASI SDK...`);
|
||||
await $`tar -xzv -C ${wasiDir} --strip-components=1 < ${bytes}`;
|
||||
|
||||
console.log(`Downloading libclang_rt.builtins-wasm32-wasi-${WASI_VERSION}.0...`);
|
||||
console.log(`Downloading libclang_rt.builtins-wasm32-wasi-${WASI_VERSION}...`);
|
||||
const rtResponse = await fetch(rtURL);
|
||||
|
||||
if (!rtResponse.ok) {
|
||||
|
|
@ -56,7 +63,7 @@ const getCompiler = async (): Promise<CompilerWithFlags> => {
|
|||
|
||||
const rtBytes = await rtResponse.bytes();
|
||||
|
||||
console.log(`Extracting libclang_rt.builtins-wasm32-wasi-${WASI_VERSION}.0...`);
|
||||
console.log(`Extracting libclang_rt.builtins-wasm32-wasi-${WASI_VERSION}...`);
|
||||
await $`tar -xzv -C ${wasiDir} --strip-components=1 < ${rtBytes}`;
|
||||
}
|
||||
|
||||
|
|
@ -67,7 +74,58 @@ const getCompiler = async (): Promise<CompilerWithFlags> => {
|
|||
`--sysroot=${path.resolve(wasiDir, 'share', 'wasi-sysroot')}`,
|
||||
];
|
||||
|
||||
return cc;
|
||||
return cc;
|
||||
})();
|
||||
return compilerPromise;
|
||||
};
|
||||
|
||||
async function mtimeMs(p: string): Promise<number> {
|
||||
return fs.stat(p).then(s => s.mtimeMs).catch(() => 0);
|
||||
}
|
||||
|
||||
async function needsRebuild(output: string, inputs: string[], flags?: string[]): Promise<boolean> {
|
||||
const [outTime, storedFlags] = await Promise.all([
|
||||
mtimeMs(output),
|
||||
flags ? Bun.file(output + '.flags').text().catch(() => null) : Promise.resolve(null),
|
||||
]);
|
||||
if (outTime === 0) return true;
|
||||
if (flags && storedFlags !== flags.join('\0')) return true;
|
||||
const inTimes = await Promise.all(inputs.map(mtimeMs));
|
||||
return inTimes.some(t => t > outTime);
|
||||
}
|
||||
|
||||
async function saveFlags(output: string, flags: string[]): Promise<void> {
|
||||
await Bun.write(output + '.flags', flags.join('\0'));
|
||||
}
|
||||
|
||||
async function compileLibGroup(
|
||||
cc: CompilerWithFlags,
|
||||
flags: string[],
|
||||
sources: string[],
|
||||
std: string[],
|
||||
output: string,
|
||||
): Promise<void> {
|
||||
if (sources.length === 0) return;
|
||||
|
||||
// Per-file object dir derived from output name (dist/lib.c.o -> dist/lib.c/)
|
||||
const objDir = output.replace(/\.o$/, '');
|
||||
await fs.mkdir(objDir, { recursive: true });
|
||||
const objPaths = sources.map(src => path.resolve(objDir, path.basename(src) + '.o'));
|
||||
|
||||
// Compile only changed files, in parallel
|
||||
const compilationFlags = [...flags, ...std];
|
||||
await Promise.all(sources.map(async (src, i) => {
|
||||
if (await needsRebuild(objPaths[i], [src], compilationFlags)) {
|
||||
const result = await $`${cc.cc} -c ${flags} ${std} -o ${objPaths[i]} ${src}`;
|
||||
if (result.exitCode !== 0) throw new Error(`Compile failed: ${src}`);
|
||||
await saveFlags(objPaths[i], compilationFlags);
|
||||
}
|
||||
}));
|
||||
|
||||
// Partial link if any object is newer than combined output
|
||||
if (!(await needsRebuild(output, objPaths))) return;
|
||||
const partialResult = await $`${cc.cc} -r -o ${output} ${objPaths}`;
|
||||
if (partialResult.exitCode !== 0) throw new Error(`Partial link failed for ${output}`);
|
||||
}
|
||||
|
||||
async function instantiate(url: string) {
|
||||
|
|
@ -189,7 +247,9 @@ const wasmPlugin = ({ production }: WasmLoaderConfig = {}): BunPlugin => {
|
|||
name: "WASM loader",
|
||||
async setup(build) {
|
||||
build.onLoad({ filter: /\.(c(pp)?|wasm)$/ }, async (args) => {
|
||||
let wasmPath = path.resolve(import.meta.dir, '..', '..', 'dist', 'tmp.wasm');
|
||||
const distDir = path.resolve(import.meta.dir, '..', '..', 'dist');
|
||||
const inputBasename = path.basename(args.path);
|
||||
let wasmPath = path.resolve(distDir, inputBasename.replace(/\.(c|cpp)$/, '.wasm'));
|
||||
let jsContent: string = `
|
||||
${instantiate}
|
||||
const module = await instantiate(new URL($WASM$));
|
||||
|
|
@ -199,11 +259,10 @@ const wasmPlugin = ({ production }: WasmLoaderConfig = {}): BunPlugin => {
|
|||
if (args.path.endsWith('.wasm')) {
|
||||
wasmPath = args.path;
|
||||
} else {
|
||||
await fs.mkdir(distDir, { recursive: true });
|
||||
|
||||
const buildAssets = path.resolve(import.meta.dir, '..', 'assets');
|
||||
const include = `${buildAssets}/include`;
|
||||
const glob = new Bun.Glob(`${buildAssets}/lib/**/*.c`);
|
||||
const stdlib = await Array.fromAsync(glob.scan());
|
||||
const objPath = wasmPath + '.o';
|
||||
const cc = await getCompiler();
|
||||
|
||||
const features = [
|
||||
|
|
@ -223,6 +282,10 @@ const wasmPlugin = ({ production }: WasmLoaderConfig = {}): BunPlugin => {
|
|||
production ? '-O3' : '-O0',
|
||||
'-flto',
|
||||
'-fno-exceptions',
|
||||
'-fno-stack-protector',
|
||||
'-ffast-math',
|
||||
'-ffunction-sections',
|
||||
'-fdata-sections',
|
||||
'-Wall',
|
||||
'-Wextra',
|
||||
'-Wpedantic',
|
||||
|
|
@ -233,37 +296,74 @@ const wasmPlugin = ({ production }: WasmLoaderConfig = {}): BunPlugin => {
|
|||
...features,
|
||||
'-I', include,
|
||||
];
|
||||
const std = args.path.endsWith('.cpp') ? '-std=gnu++26' : '-std=gnu23';
|
||||
const compileResult = await $`${cc.cc} -c ${flags} ${std} -o ${objPath} ${args.path}`;
|
||||
|
||||
if (compileResult.exitCode !== 0) {
|
||||
throw new Error('Compile failed, check output');
|
||||
}
|
||||
// Lib flags without -flto so clang -r (partial link) works on regular wasm objects
|
||||
const libFlags = flags.filter(f => f !== '-flto');
|
||||
|
||||
const symbolsResult = await $`${cc.nm} ${objPath}`.quiet();
|
||||
const std = args.path.endsWith('.cpp') ? STD_CPP : STD_C;
|
||||
|
||||
if (symbolsResult.exitCode !== 0) {
|
||||
throw new Error('nm failed, check output');
|
||||
}
|
||||
const inputObjPath = path.resolve(distDir, inputBasename + '.o');
|
||||
const libCObjPath = path.resolve(distDir, 'lib.c.o');
|
||||
const libCppObjPath = path.resolve(distDir, 'lib.cpp.o');
|
||||
|
||||
const exports = symbolsResult.text().split('\n')
|
||||
.map(l => l.match(/^-+ T ([a-z][a-z0-9_]*)/i))
|
||||
.filter(m => m != null)
|
||||
.map(m => m[1]);
|
||||
const libCGlob = new Bun.Glob(`${buildAssets}/lib/**/*.c`);
|
||||
const libCppGlob = new Bun.Glob(`${buildAssets}/lib/**/*.cpp`);
|
||||
const [libCFiles, libCppFiles] = await Promise.all([
|
||||
Array.fromAsync(libCGlob.scan()),
|
||||
Array.fromAsync(libCppGlob.scan()),
|
||||
]);
|
||||
|
||||
const linkFlags = [
|
||||
'--strip-all',
|
||||
'--lto-O3',
|
||||
'--no-entry',
|
||||
'--import-memory',
|
||||
'--export-table',
|
||||
...exports.flatMap(e => ['--export', e]),
|
||||
].map(f => `-Wl,${f}`);
|
||||
// Compile all three units in parallel
|
||||
await Promise.all([
|
||||
// 1. Input file -> dist/{filename}.o
|
||||
(async () => {
|
||||
const compilationFlags = [...flags, ...std];
|
||||
if (!await needsRebuild(inputObjPath, [args.path], compilationFlags)) return;
|
||||
const result = await $`${cc.cc} -c ${flags} ${std} -o ${inputObjPath} ${args.path}`;
|
||||
if (result.exitCode !== 0) throw new Error('Compile failed, check output');
|
||||
await saveFlags(inputObjPath, compilationFlags);
|
||||
})(),
|
||||
// 2. lib/**/*.c -> dist/lib.c.o
|
||||
compileLibGroup(cc, libFlags, libCFiles, STD_C, libCObjPath),
|
||||
// 3. lib/**/*.cpp -> dist/lib.cpp.o
|
||||
compileLibGroup(cc, libFlags, libCppFiles, STD_CPP, libCppObjPath),
|
||||
]);
|
||||
|
||||
const linkResult = await $`${cc.cc} ${flags} -std=gnu23 ${linkFlags} -lstdc++ -nostartfiles -o ${wasmPath} ${objPath} ${stdlib}`;
|
||||
// 4. Link if any input object is newer than the wasm output
|
||||
const libObjs = [
|
||||
libCFiles.length > 0 ? libCObjPath : null,
|
||||
libCppFiles.length > 0 ? libCppObjPath : null,
|
||||
].filter((x): x is string => x !== null);
|
||||
|
||||
if (linkResult.exitCode !== 0) {
|
||||
throw new Error('Link failed, check output');
|
||||
const allObjs = [inputObjPath, ...libObjs];
|
||||
|
||||
if (await needsRebuild(wasmPath, allObjs)) {
|
||||
const symbolsResult = await $`${cc.nm} ${inputObjPath}`.quiet();
|
||||
|
||||
if (symbolsResult.exitCode !== 0) {
|
||||
throw new Error('nm failed, check output');
|
||||
}
|
||||
|
||||
const exports = symbolsResult.text().split('\n')
|
||||
.map(l => l.match(/^-+ T ([a-z][a-z0-9_]*)/i))
|
||||
.filter(m => m != null)
|
||||
.map(m => m[1]);
|
||||
|
||||
const linkFlags = [
|
||||
'--strip-all',
|
||||
'--lto-O3',
|
||||
'--gc-sections',
|
||||
'--no-entry',
|
||||
'--import-memory',
|
||||
'--export-table',
|
||||
...exports.flatMap(e => ['--export', e]),
|
||||
].map(f => `-Wl,${f}`);
|
||||
|
||||
const linkResult = await $`${cc.cc} ${flags} ${linkFlags} -o ${wasmPath} -lstdc++ -nostartfiles ${allObjs}`;
|
||||
|
||||
if (linkResult.exitCode !== 0) {
|
||||
throw new Error('Link failed, check output');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -282,4 +382,4 @@ const wasmPlugin = ({ production }: WasmLoaderConfig = {}): BunPlugin => {
|
|||
return p;
|
||||
};
|
||||
|
||||
export default wasmPlugin;
|
||||
export default wasmPlugin;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <format>
|
||||
#include <iostream>
|
||||
#include <js.h>
|
||||
#include <string>
|
||||
|
|
@ -85,14 +84,4 @@ JS_EXPORT uint64_t awoo() {
|
|||
std::cout << "sum = " << sum << "\ncarry = " << carry << std::endl;
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
JS_EXPORT void play() {
|
||||
auto awoo = std::format("{2} {1}{0}!\n", 23, "C++", "Hello");
|
||||
std::puts(awoo.c_str());
|
||||
}
|
||||
|
||||
int main() {
|
||||
awoo();
|
||||
play();
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue