Compile libs/input in parallel

build/plugins/wasmPlugin.ts

@@ -13,12 +13,19 @@ interface CompilerWithFlags {
     flags: string[];
 }
 
+const STD_C = ['-std=gnu23'];
+const STD_CPP = ['-std=gnu++26', '-fno-rtti'];
+
 const WASI_VERSION = '32.0';
 const WASI_MAJOR = WASI_VERSION.split('.')[0];
 const wasiArchiveURL = `https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_MAJOR}/wasi-sdk-${WASI_VERSION}-x86_64-linux.tar.gz`;
 const rtURL = `https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_MAJOR}/libclang_rt-${WASI_VERSION}.tar.gz`;
 
-const getCompiler = async (): Promise<CompilerWithFlags> => {
+let compilerPromise: Promise<CompilerWithFlags> | null = null;
+
+const getCompiler = (): Promise<CompilerWithFlags> => {
+    if (compilerPromise) return compilerPromise;
+    compilerPromise = (async (): Promise<CompilerWithFlags> => {
     const wasiDir = path.resolve(import.meta.dir, '..', '..', 'dist', 'wasi');
     const cc: CompilerWithFlags = {
         cc: 'clang',
@@ -47,7 +54,7 @@ const getCompiler = async (): Promise<CompilerWithFlags> => {
         console.log(`Extracting WASI SDK...`);
         await $`tar -xzv -C ${wasiDir} --strip-components=1 < ${bytes}`;
 
-        console.log(`Downloading libclang_rt.builtins-wasm32-wasi-${WASI_VERSION}.0...`);
+        console.log(`Downloading libclang_rt.builtins-wasm32-wasi-${WASI_VERSION}...`);
         const rtResponse = await fetch(rtURL);
 
         if (!rtResponse.ok) {
@@ -56,7 +63,7 @@ const getCompiler = async (): Promise<CompilerWithFlags> => {
 
         const rtBytes = await rtResponse.bytes();
 
-        console.log(`Extracting libclang_rt.builtins-wasm32-wasi-${WASI_VERSION}.0...`);
+        console.log(`Extracting libclang_rt.builtins-wasm32-wasi-${WASI_VERSION}...`);
         await $`tar -xzv -C ${wasiDir} --strip-components=1 < ${rtBytes}`;
     }
 
@@ -67,7 +74,58 @@ const getCompiler = async (): Promise<CompilerWithFlags> => {
         `--sysroot=${path.resolve(wasiDir, 'share', 'wasi-sysroot')}`,
     ];
 
-    return cc;
+        return cc;
+    })();
+    return compilerPromise;
+};
+
+async function mtimeMs(p: string): Promise<number> {
+    return fs.stat(p).then(s => s.mtimeMs).catch(() => 0);
+}
+
+async function needsRebuild(output: string, inputs: string[], flags?: string[]): Promise<boolean> {
+    const [outTime, storedFlags] = await Promise.all([
+        mtimeMs(output),
+        flags ? Bun.file(output + '.flags').text().catch(() => null) : Promise.resolve(null),
+    ]);
+    if (outTime === 0) return true;
+    if (flags && storedFlags !== flags.join('\0')) return true;
+    const inTimes = await Promise.all(inputs.map(mtimeMs));
+    return inTimes.some(t => t > outTime);
+}
+
+async function saveFlags(output: string, flags: string[]): Promise<void> {
+    await Bun.write(output + '.flags', flags.join('\0'));
+}
+
+async function compileLibGroup(
+    cc: CompilerWithFlags,
+    flags: string[],
+    sources: string[],
+    std: string[],
+    output: string,
+): Promise<void> {
+    if (sources.length === 0) return;
+
+    // Per-file object dir derived from output name (dist/lib.c.o -> dist/lib.c/)
+    const objDir = output.replace(/\.o$/, '');
+    await fs.mkdir(objDir, { recursive: true });
+    const objPaths = sources.map(src => path.resolve(objDir, path.basename(src) + '.o'));
+
+    // Compile only changed files, in parallel
+    const compilationFlags = [...flags, ...std];
+    await Promise.all(sources.map(async (src, i) => {
+        if (await needsRebuild(objPaths[i], [src], compilationFlags)) {
+            const result = await $`${cc.cc} -c ${flags} ${std} -o ${objPaths[i]} ${src}`;
+            if (result.exitCode !== 0) throw new Error(`Compile failed: ${src}`);
+            await saveFlags(objPaths[i], compilationFlags);
+        }
+    }));
+
+    // Partial link if any object is newer than combined output
+    if (!(await needsRebuild(output, objPaths))) return;
+    const partialResult = await $`${cc.cc} -r -o ${output} ${objPaths}`;
+    if (partialResult.exitCode !== 0) throw new Error(`Partial link failed for ${output}`);
 }
 
 async function instantiate(url: string) {
@@ -189,7 +247,9 @@ const wasmPlugin = ({ production }: WasmLoaderConfig = {}): BunPlugin => {
         name: "WASM loader",
         async setup(build) {
             build.onLoad({ filter: /\.(c(pp)?|wasm)$/ }, async (args) => {
-                let wasmPath = path.resolve(import.meta.dir, '..', '..', 'dist', 'tmp.wasm');
+                const distDir = path.resolve(import.meta.dir, '..', '..', 'dist');
+                const inputBasename = path.basename(args.path);
+                let wasmPath = path.resolve(distDir, inputBasename.replace(/\.(c|cpp)$/, '.wasm'));
                 let jsContent: string = `
                     ${instantiate}
                     const module = await instantiate(new URL($WASM$));
@@ -199,11 +259,10 @@ const wasmPlugin = ({ production }: WasmLoaderConfig = {}): BunPlugin => {
                 if (args.path.endsWith('.wasm')) {
                     wasmPath = args.path;
                 } else {
+                    await fs.mkdir(distDir, { recursive: true });
+
                     const buildAssets = path.resolve(import.meta.dir, '..', 'assets');
                     const include = `${buildAssets}/include`;
-                    const glob = new Bun.Glob(`${buildAssets}/lib/**/*.c`);
-                    const stdlib = await Array.fromAsync(glob.scan());
-                    const objPath = wasmPath + '.o';
                     const cc = await getCompiler();
 
                     const features = [
@@ -223,6 +282,10 @@ const wasmPlugin = ({ production }: WasmLoaderConfig = {}): BunPlugin => {
                         production ? '-O3' : '-O0',
                         '-flto',
                         '-fno-exceptions',
+                        '-fno-stack-protector',
+                        '-ffast-math',
+                        '-ffunction-sections',
+                        '-fdata-sections',
                         '-Wall',
                         '-Wextra',
                         '-Wpedantic',
@@ -233,37 +296,74 @@ const wasmPlugin = ({ production }: WasmLoaderConfig = {}): BunPlugin => {
                         ...features,
                         '-I', include,
                     ];
-                    const std = args.path.endsWith('.cpp') ? '-std=gnu++26' : '-std=gnu23';
-                    const compileResult = await $`${cc.cc} -c ${flags} ${std} -o ${objPath} ${args.path}`;
 
-                    if (compileResult.exitCode !== 0) {
+                    // Lib flags without -flto so clang -r (partial link) works on regular wasm objects
-                        throw new Error('Compile failed, check output');
+                    const libFlags = flags.filter(f => f !== '-flto');
-                    }
 
-                    const symbolsResult = await $`${cc.nm} ${objPath}`.quiet();
+                    const std = args.path.endsWith('.cpp') ? STD_CPP : STD_C;
 
-                    if (symbolsResult.exitCode !== 0) {
+                    const inputObjPath = path.resolve(distDir, inputBasename + '.o');
-                        throw new Error('nm failed, check output');
+                    const libCObjPath = path.resolve(distDir, 'lib.c.o');
-                    }
+                    const libCppObjPath = path.resolve(distDir, 'lib.cpp.o');
 
-                    const exports = symbolsResult.text().split('\n')
+                    const libCGlob = new Bun.Glob(`${buildAssets}/lib/**/*.c`);
-                        .map(l => l.match(/^-+ T ([a-z][a-z0-9_]*)/i))
+                    const libCppGlob = new Bun.Glob(`${buildAssets}/lib/**/*.cpp`);
-                        .filter(m => m != null)
+                    const [libCFiles, libCppFiles] = await Promise.all([
-                        .map(m => m[1]);
+                        Array.fromAsync(libCGlob.scan()),
+                        Array.fromAsync(libCppGlob.scan()),
+                    ]);
 
-                    const linkFlags = [
+                    // Compile all three units in parallel
-                        '--strip-all',
+                    await Promise.all([
-                        '--lto-O3',
+                        // 1. Input file -> dist/{filename}.o
-                        '--no-entry',
+                        (async () => {
-                        '--import-memory',
+                            const compilationFlags = [...flags, ...std];
-                        '--export-table',
+                            if (!await needsRebuild(inputObjPath, [args.path], compilationFlags)) return;
-                        ...exports.flatMap(e => ['--export', e]),
+                            const result = await $`${cc.cc} -c ${flags} ${std} -o ${inputObjPath} ${args.path}`;
-                    ].map(f => `-Wl,${f}`);
+                            if (result.exitCode !== 0) throw new Error('Compile failed, check output');
+                            await saveFlags(inputObjPath, compilationFlags);
+                        })(),
+                        // 2. lib/**/*.c -> dist/lib.c.o
+                        compileLibGroup(cc, libFlags, libCFiles, STD_C, libCObjPath),
+                        // 3. lib/**/*.cpp -> dist/lib.cpp.o
+                        compileLibGroup(cc, libFlags, libCppFiles, STD_CPP, libCppObjPath),
+                    ]);
 
-                    const linkResult = await $`${cc.cc} ${flags} -std=gnu23 ${linkFlags} -lstdc++ -nostartfiles -o ${wasmPath} ${objPath} ${stdlib}`;
+                    // 4. Link if any input object is newer than the wasm output
+                    const libObjs = [
+                        libCFiles.length > 0 ? libCObjPath : null,
+                        libCppFiles.length > 0 ? libCppObjPath : null,
+                    ].filter((x): x is string => x !== null);
 
-                    if (linkResult.exitCode !== 0) {
+                    const allObjs = [inputObjPath, ...libObjs];
-                        throw new Error('Link failed, check output');
+
+                    if (await needsRebuild(wasmPath, allObjs)) {
+                        const symbolsResult = await $`${cc.nm} ${inputObjPath}`.quiet();
+
+                        if (symbolsResult.exitCode !== 0) {
+                            throw new Error('nm failed, check output');
+                        }
+
+                        const exports = symbolsResult.text().split('\n')
+                            .map(l => l.match(/^-+ T ([a-z][a-z0-9_]*)/i))
+                            .filter(m => m != null)
+                            .map(m => m[1]);
+
+                        const linkFlags = [
+                            '--strip-all',
+                            '--lto-O3',
+                            '--gc-sections',
+                            '--no-entry',
+                            '--import-memory',
+                            '--export-table',
+                            ...exports.flatMap(e => ['--export', e]),
+                        ].map(f => `-Wl,${f}`);
+
+                        const linkResult = await $`${cc.cc} ${flags} ${linkFlags} -o ${wasmPath} -lstdc++ -nostartfiles ${allObjs}`;
+
+                        if (linkResult.exitCode !== 0) {
+                            throw new Error('Link failed, check output');
+                        }
                     }
                 }
 

src/games/playground/awoo.cpp

@@ -1,7 +1,6 @@
 #include <cstddef>
 #include <cstdint>
 #include <cstdlib>
-#include <format>
 #include <iostream>
 #include <js.h>
 #include <string>
@@ -86,13 +85,3 @@ JS_EXPORT uint64_t awoo() {
 
     return sum;
 }
-
-JS_EXPORT void play() {
-    auto awoo = std::format("{2} {1}{0}!\n", 23, "C++", "Hello");
-    std::puts(awoo.c_str());
-}
-
-int main() {
-    awoo();
-    play();
-}