mirror of
https://github.com/vega-org/vega-providers.git
synced 2026-04-17 15:41:45 +00:00
feat: integrate esbuild for bundling providers and update dependencies
This commit is contained in:
Himanshu
528
build-bundled.js
528
build-bundled.js
@@ -1,432 +1,172 @@
|
||||
const esbuild = require("esbuild");
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const { execSync } = require("child_process");
|
||||
const { minify } = require("terser");
|
||||
|
||||
// Build configuration
|
||||
const PROVIDERS_DIR = "./providers";
|
||||
const DIST_DIR = "./dist";
|
||||
const TEMP_DIR = "./temp-build";
|
||||
const SKIP_MINIFY = process.env.SKIP_MINIFY === "true";
|
||||
|
||||
// Colors for console output
|
||||
const colors = {
|
||||
reset: "\x1b[0m",
|
||||
bright: "\x1b[1m",
|
||||
green: "\x1b[32m",
|
||||
red: "\x1b[31m",
|
||||
yellow: "\x1b[33m",
|
||||
blue: "\x1b[34m",
|
||||
magenta: "\x1b[35m",
|
||||
cyan: "\x1b[36m",
|
||||
};
|
||||
// Find all provider directories
|
||||
const providersDir = path.join(__dirname, "providers");
|
||||
const providerDirs = fs
|
||||
.readdirSync(providersDir, { withFileTypes: true })
|
||||
.filter(
|
||||
(dirent) =>
|
||||
dirent.isDirectory() && !dirent.name.startsWith(".") && dirent.name !== "extractors",
|
||||
)
|
||||
.map((dirent) => dirent.name);
|
||||
|
||||
const log = {
|
||||
info: (msg) => console.log(`${colors.blue}ℹ${colors.reset} ${msg}`),
|
||||
success: (msg) => console.log(`${colors.green}✅${colors.reset} ${msg}`),
|
||||
error: (msg) => console.log(`${colors.red}❌${colors.reset} ${msg}`),
|
||||
warning: (msg) => console.log(`${colors.yellow}⚠️${colors.reset} ${msg}`),
|
||||
build: (msg) => console.log(`${colors.magenta}🔨${colors.reset} ${msg}`),
|
||||
file: (msg) => console.log(`${colors.cyan}📄${colors.reset} ${msg}`),
|
||||
};
|
||||
console.log(`Found ${providerDirs.length} providers to build`);
|
||||
|
||||
/**
|
||||
* Bundled provider builder - creates self-contained JS files without imports
|
||||
*/
|
||||
class BundledProviderBuilder {
|
||||
constructor() {
|
||||
this.startTime = Date.now();
|
||||
this.providers = [];
|
||||
async function buildProvider(providerName) {
|
||||
const providerPath = path.join(providersDir, providerName);
|
||||
const distPath = path.join(__dirname, "dist", providerName);
|
||||
|
||||
// Create dist directory
|
||||
if (!fs.existsSync(distPath)) {
|
||||
fs.mkdirSync(distPath, { recursive: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean the dist and temp directories
|
||||
*/
|
||||
cleanDirs() {
|
||||
if (fs.existsSync(DIST_DIR)) {
|
||||
fs.rmSync(DIST_DIR, { recursive: true, force: true });
|
||||
const modules = ["catalog", "posts", "meta", "stream", "episodes"];
|
||||
const results = [];
|
||||
|
||||
for (const moduleName of modules) {
|
||||
const modulePath = path.join(providerPath, `${moduleName}.ts`);
|
||||
|
||||
if (!fs.existsSync(modulePath)) {
|
||||
continue;
|
||||
}
|
||||
fs.mkdirSync(DIST_DIR, { recursive: true });
|
||||
|
||||
if (fs.existsSync(TEMP_DIR)) {
|
||||
fs.rmSync(TEMP_DIR, { recursive: true, force: true });
|
||||
}
|
||||
fs.mkdirSync(TEMP_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover all provider directories (excluding extractors and utility files)
|
||||
*/
|
||||
discoverProviders() {
|
||||
const items = fs.readdirSync(PROVIDERS_DIR, { withFileTypes: true });
|
||||
const excludeDirs = ["extractors", "extractors copy"];
|
||||
|
||||
this.providers = items
|
||||
.filter((item) => item.isDirectory())
|
||||
.filter((item) => !item.name.startsWith("."))
|
||||
.filter((item) => !excludeDirs.includes(item.name))
|
||||
.map((item) => item.name);
|
||||
|
||||
log.info(
|
||||
`Found ${this.providers.length} providers: ${this.providers.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compile TypeScript to JavaScript first
|
||||
*/
|
||||
compileTypeScript() {
|
||||
log.build("Compiling TypeScript files...");
|
||||
|
||||
try {
|
||||
execSync("npx tsc", {
|
||||
stdio: "pipe",
|
||||
encoding: "utf8",
|
||||
// Use esbuild to bundle the module
|
||||
const result = await esbuild.build({
|
||||
entryPoints: [modulePath],
|
||||
bundle: true,
|
||||
platform: "node",
|
||||
format: "cjs",
|
||||
target: "es2015",
|
||||
write: false,
|
||||
external: [], // Bundle everything
|
||||
minify: false, // We'll minify separately if needed
|
||||
keepNames: true,
|
||||
treeShaking: true,
|
||||
outfile: `${moduleName}.js`,
|
||||
});
|
||||
return true;
|
||||
} catch (error) {
|
||||
log.error("TypeScript compilation failed:");
|
||||
if (error.stdout) console.log(error.stdout);
|
||||
if (error.stderr) console.log(error.stderr);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Bundle each provider module to be self-contained
|
||||
* This inlines all imports from extractors into the provider files
|
||||
*/
|
||||
bundleProviders() {
|
||||
log.build("Bundling provider modules...");
|
||||
let code = result.outputFiles[0].text;
|
||||
|
||||
for (const provider of this.providers) {
|
||||
const providerDistDir = path.join(DIST_DIR, provider);
|
||||
|
||||
if (!fs.existsSync(providerDistDir)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const files = [
|
||||
"stream.js",
|
||||
"catalog.js",
|
||||
"posts.js",
|
||||
"meta.js",
|
||||
"episodes.js",
|
||||
];
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = path.join(providerDistDir, file);
|
||||
if (fs.existsSync(filePath)) {
|
||||
this.bundleFile(filePath, provider);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Bundle a single file by inlining all local imports
|
||||
*/
|
||||
bundleFile(filePath, provider) {
|
||||
let content = fs.readFileSync(filePath, "utf8");
|
||||
|
||||
// Find all require statements - both destructuring and non-destructuring patterns
|
||||
// Pattern 1: const { x, y } = require("path")
|
||||
const destructuringRegex =
|
||||
/(?:const|let|var)\s+\{([^}]+)\}\s*=\s*require\s*\(\s*["']([^"']+)["']\s*\);?/g;
|
||||
// Pattern 2: const hubcloud_1 = require("path")
|
||||
const simpleRequireRegex =
|
||||
/(?:const|let|var)\s+(\w+)\s*=\s*require\s*\(\s*["']([^"']+)["']\s*\);?/g;
|
||||
|
||||
const imports = [];
|
||||
let match;
|
||||
|
||||
while ((match = destructuringRegex.exec(content)) !== null) {
|
||||
imports.push({
|
||||
full: match[0],
|
||||
names: match[1],
|
||||
varName: null,
|
||||
path: match[2],
|
||||
isDestructuring: true,
|
||||
});
|
||||
}
|
||||
|
||||
while ((match = simpleRequireRegex.exec(content)) !== null) {
|
||||
// Skip if already matched by destructuring regex
|
||||
if (imports.some((i) => i.full === match[0])) continue;
|
||||
imports.push({
|
||||
full: match[0],
|
||||
names: null,
|
||||
varName: match[1],
|
||||
path: match[2],
|
||||
isDestructuring: false,
|
||||
});
|
||||
}
|
||||
|
||||
// Process each import
|
||||
for (const imp of imports) {
|
||||
// Skip external modules (axios, cheerio, etc.) - they come from context
|
||||
if (!imp.path.startsWith(".") && !imp.path.startsWith("/")) {
|
||||
// Remove the require statement for external modules
|
||||
content = content.replace(imp.full, `// External: ${imp.path}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Resolve the import path
|
||||
const importDir = path.dirname(filePath);
|
||||
let resolvedPath = path.resolve(importDir, imp.path);
|
||||
|
||||
// Add .js extension if needed
|
||||
if (!resolvedPath.endsWith(".js")) {
|
||||
resolvedPath += ".js";
|
||||
}
|
||||
|
||||
if (fs.existsSync(resolvedPath)) {
|
||||
// Read the imported file
|
||||
let importedContent = fs.readFileSync(resolvedPath, "utf8");
|
||||
|
||||
// Remove exports.X = X pattern and just keep the functions
|
||||
importedContent = importedContent.replace(
|
||||
/exports\.\w+\s*=\s*\w+;?/g,
|
||||
"",
|
||||
// Post-process the code for React Native compatibility
|
||||
// Remove require statements for built-in modules that aren't available
|
||||
code = code.replace(/require\(['"]node:.*?['"]\)/g, "{}");
|
||||
|
||||
// Fix CommonJS exports for React Native's ProviderManager execution context
|
||||
// The ProviderManager only has `exports` object, not `module`
|
||||
|
||||
// Extract what's being exported by looking at the __export call
|
||||
// Pattern: __export(xxx_exports, { funcName: () => funcName, ... });
|
||||
const exportMatch = code.match(/__export\((\w+),\s*\{([^}]+)\}\);/);
|
||||
|
||||
if (exportMatch) {
|
||||
const exportsVar = exportMatch[1];
|
||||
const exportsContent = exportMatch[2];
|
||||
|
||||
// Parse the export entries like "funcName: () => funcName"
|
||||
const exportEntries = exportsContent
|
||||
.split(',')
|
||||
.map(entry => {
|
||||
const match = entry.trim().match(/(\w+):\s*\(\)\s*=>\s*(\w+)/);
|
||||
return match ? match[1] : null;
|
||||
})
|
||||
.filter(Boolean);
|
||||
|
||||
// Replace module.exports pattern
|
||||
code = code.replace(
|
||||
/module\.exports\s*=\s*__toCommonJS\((\w+)\);/g,
|
||||
''
|
||||
);
|
||||
|
||||
// Remove Object.defineProperty exports
|
||||
importedContent = importedContent.replace(
|
||||
/Object\.defineProperty\(exports,\s*"__esModule"[^;]+;/g,
|
||||
"",
|
||||
|
||||
// Add direct exports assignments at the end
|
||||
const directExports = exportEntries
|
||||
.map(name => `exports.${name} = ${name};`)
|
||||
.join('\n');
|
||||
|
||||
// Add the exports before the final comment
|
||||
code = code.replace(
|
||||
/\/\/ Annotate the CommonJS export names for ESM import in node:/,
|
||||
`${directExports}\n// Annotate the CommonJS export names for ESM import in node:`
|
||||
);
|
||||
|
||||
// Remove require statements from imported file too (they use context)
|
||||
importedContent = importedContent.replace(
|
||||
/(?:const|let|var)\s+\{[^}]+\}\s*=\s*require\s*\(\s*["'][^"']+["']\s*\);?/g,
|
||||
"",
|
||||
);
|
||||
importedContent = importedContent.replace(
|
||||
/(?:const|let|var)\s+\w+\s*=\s*require\s*\(\s*["'][^"']+["']\s*\);?/g,
|
||||
"",
|
||||
);
|
||||
|
||||
// Clean up the content
|
||||
importedContent = importedContent.replace(/"use strict";?/g, "");
|
||||
|
||||
// Check if this is an extractor file
|
||||
if (
|
||||
imp.path.includes("extractor") ||
|
||||
resolvedPath.includes("extractor")
|
||||
) {
|
||||
// Insert the extractor function at the top of the file
|
||||
content = content.replace(
|
||||
imp.full,
|
||||
`// Inlined from: ${imp.path}\n${importedContent.trim()}`,
|
||||
);
|
||||
|
||||
// For non-destructuring imports, we need to replace function calls
|
||||
// TypeScript outputs: (0, hubcloud_1.hubcloudExtractor)(...)
|
||||
// We need to replace with just: hubcloudExtractor(...)
|
||||
if (!imp.isDestructuring && imp.varName) {
|
||||
// Find the exported function name - look for exports.funcName or function funcNameExtractor
|
||||
// The exports pattern looks like: exports.hubcloudExtractor = hubcloudExtractor;
|
||||
const exportsMatch = importedContent.match(
|
||||
/exports\.(\w+Extractor)\s*=/,
|
||||
);
|
||||
// Also try matching the function definition directly
|
||||
const funcDefMatch = importedContent.match(
|
||||
/function\s+(\w+Extractor)\s*\(/,
|
||||
);
|
||||
const funcName = exportsMatch?.[1] || funcDefMatch?.[1];
|
||||
|
||||
if (funcName) {
|
||||
// Replace (0, varName.funcName) or (0,varName.funcName) with just funcName
|
||||
const callPattern = new RegExp(
|
||||
`\\(0,\\s*${imp.varName}\\.${funcName}\\)`,
|
||||
"g",
|
||||
);
|
||||
content = content.replace(callPattern, funcName);
|
||||
// Also replace varName.funcName (without the (0, ) wrapper)
|
||||
const simpleCallPattern = new RegExp(
|
||||
`${imp.varName}\\.${funcName}`,
|
||||
"g",
|
||||
);
|
||||
content = content.replace(simpleCallPattern, funcName);
|
||||
}
|
||||
}
|
||||
} else if (imp.path.includes("types")) {
|
||||
// Types are not needed at runtime, just remove the import
|
||||
content = content.replace(imp.full, `// Types removed: ${imp.path}`);
|
||||
} else {
|
||||
// Other local imports - inline them
|
||||
content = content.replace(
|
||||
imp.full,
|
||||
`// Inlined from: ${imp.path}\n${importedContent.trim()}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// File doesn't exist, comment out the import
|
||||
content = content.replace(imp.full, `// Not found: ${imp.path}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up the content
|
||||
content = content.replace(/"use strict";?/g, "");
|
||||
content = content.replace(
|
||||
/Object\.defineProperty\(exports,\s*"__esModule"[^;]+;/g,
|
||||
"",
|
||||
);
|
||||
|
||||
// Write the bundled file
|
||||
fs.writeFileSync(filePath, content);
|
||||
}
|
||||
|
||||
/**
|
||||
* Minify all JavaScript files
|
||||
*/
|
||||
async minifyFiles() {
|
||||
const keepConsole = process.env.KEEP_CONSOLE === "true";
|
||||
log.build(
|
||||
`Minifying JavaScript files... ${
|
||||
keepConsole ? "(keeping console logs)" : "(removing console logs)"
|
||||
}`,
|
||||
);
|
||||
|
||||
const minifyFile = async (filePath) => {
|
||||
try {
|
||||
const code = fs.readFileSync(filePath, "utf8");
|
||||
const result = await minify(code, {
|
||||
|
||||
// Also handle the "0 && (module.exports = {...})" pattern at the end
|
||||
code = code.replace(/0\s*&&\s*\(module\.exports\s*=\s*\{[^}]*\}\);?/g, "");
|
||||
|
||||
// Minify if not skipped
|
||||
if (!SKIP_MINIFY) {
|
||||
const minified = await minify(code, {
|
||||
compress: {
|
||||
drop_console: !keepConsole,
|
||||
drop_debugger: true,
|
||||
pure_funcs: keepConsole
|
||||
? ["console.debug"]
|
||||
: [
|
||||
"console.debug",
|
||||
"console.log",
|
||||
"console.info",
|
||||
"console.warn",
|
||||
],
|
||||
drop_console: false,
|
||||
passes: 2,
|
||||
},
|
||||
mangle: {
|
||||
keep_fnames: false,
|
||||
},
|
||||
mangle: false,
|
||||
format: {
|
||||
comments: false,
|
||||
},
|
||||
});
|
||||
|
||||
if (result.code) {
|
||||
fs.writeFileSync(filePath, result.code);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
} catch (error) {
|
||||
log.error(`Error minifying ${filePath}: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const findJsFiles = (dir) => {
|
||||
const files = [];
|
||||
if (!fs.existsSync(dir)) return files;
|
||||
|
||||
const items = fs.readdirSync(dir, { withFileTypes: true });
|
||||
for (const item of items) {
|
||||
const fullPath = path.join(dir, item.name);
|
||||
if (item.isDirectory()) {
|
||||
files.push(...findJsFiles(fullPath));
|
||||
} else if (item.isFile() && item.name.endsWith(".js")) {
|
||||
files.push(fullPath);
|
||||
if (minified.code) {
|
||||
code = minified.code;
|
||||
}
|
||||
}
|
||||
return files;
|
||||
};
|
||||
|
||||
const jsFiles = findJsFiles(DIST_DIR);
|
||||
let minifiedCount = 0;
|
||||
let totalSizeBefore = 0;
|
||||
let totalSizeAfter = 0;
|
||||
// Write the output
|
||||
const outputPath = path.join(distPath, `${moduleName}.js`);
|
||||
fs.writeFileSync(outputPath, code);
|
||||
|
||||
for (const filePath of jsFiles) {
|
||||
const statsBefore = fs.statSync(filePath);
|
||||
totalSizeBefore += statsBefore.size;
|
||||
results.push({
|
||||
moduleName,
|
||||
size: code.length,
|
||||
});
|
||||
|
||||
const success = await minifyFile(filePath);
|
||||
if (success) {
|
||||
const statsAfter = fs.statSync(filePath);
|
||||
totalSizeAfter += statsAfter.size;
|
||||
minifiedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
const compressionRatio =
|
||||
totalSizeBefore > 0
|
||||
? (
|
||||
((totalSizeBefore - totalSizeAfter) / totalSizeBefore) *
|
||||
100
|
||||
).toFixed(1)
|
||||
: 0;
|
||||
|
||||
log.success(
|
||||
`Minified ${minifiedCount}/${jsFiles.length} files. ` +
|
||||
`Size reduced by ${compressionRatio}% (${totalSizeBefore} → ${totalSizeAfter} bytes)`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up temp directory
|
||||
*/
|
||||
cleanup() {
|
||||
if (fs.existsSync(TEMP_DIR)) {
|
||||
fs.rmSync(TEMP_DIR, { recursive: true, force: true });
|
||||
console.log(
|
||||
`✓ ${providerName}/${moduleName}.js (${(code.length / 1024).toFixed(1)}kb)`,
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(`✗ Error building ${providerName}/${moduleName}:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build everything
|
||||
*/
|
||||
async build() {
|
||||
const isWatchMode = process.env.NODE_ENV === "development";
|
||||
|
||||
if (isWatchMode) {
|
||||
console.log(
|
||||
`\n${colors.cyan}🔄 Auto-build triggered${colors.reset} ${new Date().toLocaleTimeString()}`,
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
`\n${colors.bright}🚀 Starting bundled provider build...${colors.reset}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
this.cleanDirs();
|
||||
this.discoverProviders();
|
||||
|
||||
const compiled = this.compileTypeScript();
|
||||
if (!compiled) {
|
||||
log.error("Build failed due to compilation errors");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
this.bundleProviders();
|
||||
|
||||
if (!process.env.SKIP_MINIFY) {
|
||||
await this.minifyFiles();
|
||||
} else {
|
||||
log.info("Skipping minification (SKIP_MINIFY=true)");
|
||||
}
|
||||
|
||||
this.cleanup();
|
||||
|
||||
const buildTime = Date.now() - this.startTime;
|
||||
log.success(`Build completed in ${buildTime}ms`);
|
||||
|
||||
if (isWatchMode) {
|
||||
console.log(`${colors.green}👀 Watching for changes...${colors.reset}\n`);
|
||||
} else {
|
||||
console.log(
|
||||
`${colors.bright}✨ Build completed successfully!${colors.reset}\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return { providerName, modules: results };
|
||||
}
|
||||
|
||||
// Run the build
|
||||
const builder = new BundledProviderBuilder();
|
||||
builder.build().catch((error) => {
|
||||
async function buildAll() {
|
||||
const startTime = Date.now();
|
||||
console.log(`Building providers${SKIP_MINIFY ? " (without minification)" : ""}...\n`);
|
||||
|
||||
// Clear dist directory
|
||||
const distDir = path.join(__dirname, "dist");
|
||||
if (fs.existsSync(distDir)) {
|
||||
fs.rmSync(distDir, { recursive: true, force: true });
|
||||
}
|
||||
fs.mkdirSync(distDir, { recursive: true });
|
||||
|
||||
// Build all providers
|
||||
const results = await Promise.all(providerDirs.map(buildProvider));
|
||||
|
||||
const totalModules = results.reduce((sum, r) => sum + r.modules.length, 0);
|
||||
const totalSize = results.reduce(
|
||||
(sum, r) => sum + r.modules.reduce((s, m) => s + m.size, 0),
|
||||
0,
|
||||
);
|
||||
|
||||
const endTime = Date.now();
|
||||
console.log(
|
||||
`\n✓ Built ${totalModules} modules from ${providerDirs.length} providers in ${((endTime - startTime) / 1000).toFixed(2)}s`,
|
||||
);
|
||||
console.log(` Total size: ${(totalSize / 1024).toFixed(1)}kb`);
|
||||
}
|
||||
|
||||
buildAll().catch((error) => {
|
||||
console.error("Build failed:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user