多线程编译自适应(4核及以上开启) #4

Merged
yutent merged 1 commits from dev into master 2023-06-20 10:07:23 +08:00
1 changed files with 60 additions and 25 deletions

View File

@ -3,10 +3,14 @@ import { Worker, parentPort } from 'node:worker_threads'
import os from 'node:os' import os from 'node:os'
import fs from 'iofs' import fs from 'iofs'
import { compileFiles } from './compile.js'
import { defaultCustomElement } from './utils.js'
const IS_WIN = process.platform === 'win32' const IS_WIN = process.platform === 'win32'
const PREFIX = IS_WIN ? 'pages\\' : 'pages/' const PREFIX = IS_WIN ? 'pages\\' : 'pages/'
// 线程太多, 效率反而不高 // 4核(或4线程)以上的CPU, 才开启多线程编译。且线程开销太高, 开太多线程效率反而不高。
const THREADS_NUM = os.cpus().length > 4 ? 4 : os.cpus().length - 1 const CPU_CORES = os.cpus().length > 5 ? 6 : os.cpus().length
const THREADS_NUM = CPU_CORES > 3 ? CPU_CORES - 1 : 0
const __filename = normalize(import.meta.url.slice(IS_WIN ? 8 : 7)) const __filename = normalize(import.meta.url.slice(IS_WIN ? 8 : 7))
const __dirname = dirname(__filename) const __dirname = dirname(__filename)
const WORKER_POOL = new Set() // 线程池 const WORKER_POOL = new Set() // 线程池
@ -62,21 +66,28 @@ export default function compile(root = '', dist = '', conf = {}, verbose) {
INJECT_SCSS, INJECT_SCSS,
LEGACY_MODE, LEGACY_MODE,
// 线程通讯无法传递函数类型, 需要转为字符串, 之后再转回来 // 线程通讯无法传递函数类型, 需要转为字符串, 之后再转回来
isCustomElement: isCustomElement ? isCustomElement.toString() : null isCustomElement:
THREADS_NUM > 0
? isCustomElement
? isCustomElement.toString()
: null
: isCustomElement || defaultCustomElement
} }
// 创建线程池 // 创建线程池
for (let i = 0; i < THREADS_NUM; i++) { if (THREADS_NUM > 0) {
WORKER_POOL.add( for (let i = 0; i < THREADS_NUM; i++) {
new Worker(join(__dirname, './thread.js'), { WORKER_POOL.add(
workerData: { new Worker(join(__dirname, './thread.js'), {
options, workerData: {
verbose, options,
dist, verbose,
imports: conf.imports dist,
} imports: conf.imports
}) }
) })
)
}
} }
fs.ls(SOURCE_DIR, true).forEach(path => { fs.ls(SOURCE_DIR, true).forEach(path => {
@ -141,18 +152,34 @@ export default function compile(root = '', dist = '', conf = {}, verbose) {
list.delete(path) list.delete(path)
files.set(path, { name, ext }) files.set(path, { name, ext })
}) })
chunk.set(currentPage, { page, files }) if (THREADS_NUM > 0) {
JOBS_QUEUE.push(chunk) chunk.set(currentPage, { page, files })
doJob() JOBS_QUEUE.push(chunk)
doJob()
} else {
console.log(`正在生成 ${currentPage}.html ...`)
compileFiles(currentPage, page, files, options, {
verbose,
dist,
imports: conf.imports
})
}
} }
// 公共依赖 // 公共依赖
{ if (THREADS_NUM > 0) {
let chunk = new Map() let chunk = new Map()
chunk.set('', { page: null, files: list }) chunk.set('', { page: null, files: list })
JOBS_QUEUE.push(chunk) JOBS_QUEUE.push(chunk)
doJob() doJob()
} else {
console.log('\n正在解析公共依赖 ...')
compileFiles('', null, list, options, {
verbose,
dist,
imports: conf.imports
})
} }
} else { } else {
// 每个线程处理的文件数 // 每个线程处理的文件数
@ -164,15 +191,23 @@ export default function compile(root = '', dist = '', conf = {}, verbose) {
console.log(`正在生成 ${currentPage}.html ...`) console.log(`正在生成 ${currentPage}.html ...`)
for (let i = 0; i < THREADS_NUM; i++) { if (THREADS_NUM > 0) {
let start = i * chunkSize for (let i = 0; i < THREADS_NUM; i++) {
let end = start + chunkSize let start = i * chunkSize
let chunk = new Map() let end = start + chunkSize
let chunk = new Map()
chunk.set(currentPage, { page, files: list.slice(start, end) }) chunk.set(currentPage, { page, files: list.slice(start, end) })
JOBS_QUEUE.push(chunk) JOBS_QUEUE.push(chunk)
doJob() doJob()
}
} else {
compileFiles(currentPage, page, list, options, {
verbose,
dist,
imports: conf.imports
})
} }
} }