Merge pull request #4 from bytedo/dev

多线程编译自适应(4核及以上开启)
pull/7/head
yutent 2023-06-20 10:07:23 +08:00 committed by GitHub
commit d8a783a336
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 60 additions and 25 deletions

View File

@ -3,10 +3,14 @@ import { Worker, parentPort } from 'node:worker_threads'
import os from 'node:os' import os from 'node:os'
import fs from 'iofs' import fs from 'iofs'
import { compileFiles } from './compile.js'
import { defaultCustomElement } from './utils.js'
const IS_WIN = process.platform === 'win32' const IS_WIN = process.platform === 'win32'
const PREFIX = IS_WIN ? 'pages\\' : 'pages/' const PREFIX = IS_WIN ? 'pages\\' : 'pages/'
// 线程太多, 效率反而不高 // 4核(或4线程)以上的CPU, 才开启多线程编译。且线程开销太高, 开太多线程效率反而不高。
const THREADS_NUM = os.cpus().length > 4 ? 4 : os.cpus().length - 1 const CPU_CORES = os.cpus().length > 5 ? 6 : os.cpus().length
const THREADS_NUM = CPU_CORES > 3 ? CPU_CORES - 1 : 0
const __filename = normalize(import.meta.url.slice(IS_WIN ? 8 : 7)) const __filename = normalize(import.meta.url.slice(IS_WIN ? 8 : 7))
const __dirname = dirname(__filename) const __dirname = dirname(__filename)
const WORKER_POOL = new Set() // 线程池 const WORKER_POOL = new Set() // 线程池
@ -62,10 +66,16 @@ export default function compile(root = '', dist = '', conf = {}, verbose) {
INJECT_SCSS, INJECT_SCSS,
LEGACY_MODE, LEGACY_MODE,
// 线程通讯无法传递函数类型, 需要转为字符串, 之后再转回来 // 线程通讯无法传递函数类型, 需要转为字符串, 之后再转回来
isCustomElement: isCustomElement ? isCustomElement.toString() : null isCustomElement:
THREADS_NUM > 0
? isCustomElement
? isCustomElement.toString()
: null
: isCustomElement || defaultCustomElement
} }
// 创建线程池 // 创建线程池
if (THREADS_NUM > 0) {
for (let i = 0; i < THREADS_NUM; i++) { for (let i = 0; i < THREADS_NUM; i++) {
WORKER_POOL.add( WORKER_POOL.add(
new Worker(join(__dirname, './thread.js'), { new Worker(join(__dirname, './thread.js'), {
@ -78,6 +88,7 @@ export default function compile(root = '', dist = '', conf = {}, verbose) {
}) })
) )
} }
}
fs.ls(SOURCE_DIR, true).forEach(path => { fs.ls(SOURCE_DIR, true).forEach(path => {
if (fs.isdir(path)) { if (fs.isdir(path)) {
@ -141,18 +152,34 @@ export default function compile(root = '', dist = '', conf = {}, verbose) {
list.delete(path) list.delete(path)
files.set(path, { name, ext }) files.set(path, { name, ext })
}) })
if (THREADS_NUM > 0) {
chunk.set(currentPage, { page, files }) chunk.set(currentPage, { page, files })
JOBS_QUEUE.push(chunk) JOBS_QUEUE.push(chunk)
doJob() doJob()
} else {
console.log(`正在生成 ${currentPage}.html ...`)
compileFiles(currentPage, page, files, options, {
verbose,
dist,
imports: conf.imports
})
}
} }
// 公共依赖 // 公共依赖
{ if (THREADS_NUM > 0) {
let chunk = new Map() let chunk = new Map()
chunk.set('', { page: null, files: list }) chunk.set('', { page: null, files: list })
JOBS_QUEUE.push(chunk) JOBS_QUEUE.push(chunk)
doJob() doJob()
} else {
console.log('\n正在解析公共依赖 ...')
compileFiles('', null, list, options, {
verbose,
dist,
imports: conf.imports
})
} }
} else { } else {
// 每个线程处理的文件数 // 每个线程处理的文件数
@ -164,6 +191,7 @@ export default function compile(root = '', dist = '', conf = {}, verbose) {
console.log(`正在生成 ${currentPage}.html ...`) console.log(`正在生成 ${currentPage}.html ...`)
if (THREADS_NUM > 0) {
for (let i = 0; i < THREADS_NUM; i++) { for (let i = 0; i < THREADS_NUM; i++) {
let start = i * chunkSize let start = i * chunkSize
let end = start + chunkSize let end = start + chunkSize
@ -174,6 +202,13 @@ export default function compile(root = '', dist = '', conf = {}, verbose) {
JOBS_QUEUE.push(chunk) JOBS_QUEUE.push(chunk)
doJob() doJob()
} }
} else {
compileFiles(currentPage, page, list, options, {
verbose,
dist,
imports: conf.imports
})
}
} }
process.on('exit', _ => { process.on('exit', _ => {