多线程编译自适应(4核及以上开启) #4

Merged
yutent merged 1 commits from dev into master 2023-06-20 10:07:23 +08:00
1 changed files with 60 additions and 25 deletions

View File

@ -3,10 +3,14 @@ import { Worker, parentPort } from 'node:worker_threads'
import os from 'node:os'
import fs from 'iofs'
import { compileFiles } from './compile.js'
import { defaultCustomElement } from './utils.js'
const IS_WIN = process.platform === 'win32'
const PREFIX = IS_WIN ? 'pages\\' : 'pages/'
// 线程太多, 效率反而不高
const THREADS_NUM = os.cpus().length > 4 ? 4 : os.cpus().length - 1
// 4核(或4线程)以上的CPU, 才开启多线程编译。且线程开销太高, 开太多线程效率反而不高。
const CPU_CORES = os.cpus().length > 5 ? 6 : os.cpus().length
const THREADS_NUM = CPU_CORES > 3 ? CPU_CORES - 1 : 0
const __filename = normalize(import.meta.url.slice(IS_WIN ? 8 : 7))
const __dirname = dirname(__filename)
const WORKER_POOL = new Set() // 线程池
@ -62,21 +66,28 @@ export default function compile(root = '', dist = '', conf = {}, verbose) {
INJECT_SCSS,
LEGACY_MODE,
// 线程通讯无法传递函数类型, 需要转为字符串, 之后再转回来
isCustomElement: isCustomElement ? isCustomElement.toString() : null
isCustomElement:
THREADS_NUM > 0
? isCustomElement
? isCustomElement.toString()
: null
: isCustomElement || defaultCustomElement
}
// 创建线程池
for (let i = 0; i < THREADS_NUM; i++) {
WORKER_POOL.add(
new Worker(join(__dirname, './thread.js'), {
workerData: {
options,
verbose,
dist,
imports: conf.imports
}
})
)
if (THREADS_NUM > 0) {
for (let i = 0; i < THREADS_NUM; i++) {
WORKER_POOL.add(
new Worker(join(__dirname, './thread.js'), {
workerData: {
options,
verbose,
dist,
imports: conf.imports
}
})
)
}
}
fs.ls(SOURCE_DIR, true).forEach(path => {
@ -141,18 +152,34 @@ export default function compile(root = '', dist = '', conf = {}, verbose) {
list.delete(path)
files.set(path, { name, ext })
})
chunk.set(currentPage, { page, files })
JOBS_QUEUE.push(chunk)
doJob()
if (THREADS_NUM > 0) {
chunk.set(currentPage, { page, files })
JOBS_QUEUE.push(chunk)
doJob()
} else {
console.log(`正在生成 ${currentPage}.html ...`)
compileFiles(currentPage, page, files, options, {
verbose,
dist,
imports: conf.imports
})
}
}
// 公共依赖
{
if (THREADS_NUM > 0) {
let chunk = new Map()
chunk.set('', { page: null, files: list })
JOBS_QUEUE.push(chunk)
doJob()
} else {
console.log('\n正在解析公共依赖 ...')
compileFiles('', null, list, options, {
verbose,
dist,
imports: conf.imports
})
}
} else {
// 每个线程处理的文件数
@ -164,15 +191,23 @@ export default function compile(root = '', dist = '', conf = {}, verbose) {
console.log(`正在生成 ${currentPage}.html ...`)
for (let i = 0; i < THREADS_NUM; i++) {
let start = i * chunkSize
let end = start + chunkSize
let chunk = new Map()
if (THREADS_NUM > 0) {
for (let i = 0; i < THREADS_NUM; i++) {
let start = i * chunkSize
let end = start + chunkSize
let chunk = new Map()
chunk.set(currentPage, { page, files: list.slice(start, end) })
chunk.set(currentPage, { page, files: list.slice(start, end) })
JOBS_QUEUE.push(chunk)
doJob()
JOBS_QUEUE.push(chunk)
doJob()
}
} else {
compileFiles(currentPage, page, list, options, {
verbose,
dist,
imports: conf.imports
})
}
}