Merge pull request #3 from bytedo/threads

Threads

lib/compile-vue.js

@@ -317,8 +317,9 @@ export function compileVue(file, imports, options = {}) {
       isCustomElement
     }).code.replace('export function render', 'function render')
   } catch (err) {
+    // console.log(err)
     let tmp = html[1].split('\n')
-    let line = tmp[err.loc.start.line - 1]
+    let line = tmp[err.loc?.start.line - 1]
 
     console.log('%s: %s', red('SyntaxError'), red(err.message))
     console.log(

lib/compile.js

@@ -0,0 +1,94 @@
+/**
+ * {}
+ * @author yutent<yutent.io@gmail.com>
+ * @date 2023/06/14 18:36:15
+ */
+
+import { join, dirname, parse, normalize } from 'node:path'
+import fs from 'iofs'
+import Es from 'esbuild'
+import { compileScss, parseJs, compileVue, parseHtml } from './compile-vue.js'
+
+const template = fs.cat(join(process.cwd(), 'index.html')).toString()
+
+export async function compileFiles(
+  currentPage,
+  page,
+  files,
+  options,
+  { verbose, imports, dist } = {}
+) {
+  options.currentPage = currentPage
+
+  for (let [path, it] of files) {
+    // 入口文件, 特殊处理
+    if (page && path === page.entry) {
+      let entry = fs.cat(page.entry).toString()
+
+      entry = parseJs(entry, imports, { ...options, IS_ENTRY: true })
+
+      let code = parseHtml(template, {
+        page,
+        imports,
+        entry,
+        LEGACY_MODE: options.LEGACY_MODE
+      })
+
+      fs.echo(code, join(dist, `${currentPage}.html`))
+      continue
+    }
+
+    verbose && console.log('  解析 %s ...', it.name)
+
+    let pageDir = options.IS_MPA && currentPage ? `pages/${currentPage}` : ''
+
+    switch (it.ext) {
+      case '.vue':
+        {
+          let code = compileVue(path, imports, options)
+
+          await Es.transform(code, { minify: true }).then(r => {
+            fs.echo(
+              r.code,
+              join(dist, 'assets/', pageDir, it.name.replace(/\.vue$/, '.js'))
+            )
+          })
+        }
+        break
+
+      case '.js':
+        {
+          let code = fs.cat(path)
+
+          code = parseJs(code + '', imports, options)
+
+          await Es.transform(code, { minify: true }).then(r => {
+            fs.echo(r.code, join(dist, 'assets/', pageDir, it.name))
+          })
+        }
+        break
+
+      case '.scss':
+      case '.css':
+        {
+          let target = join(
+            dist,
+            'assets/',
+            pageDir,
+            it.name.replace(/\.scss$/, '.css')
+          )
+          if (it.ext === '.css') {
+            fs.cp(path, target)
+          } else {
+            let code = compileScss(path)
+            fs.echo(code, target)
+          }
+        }
+        break
+
+      default:
+        fs.cp(path, join(dist, 'assets/', pageDir, it.name))
+        break
+    }
+  }
+}

lib/dev.js

@@ -7,7 +7,7 @@ import socket from './ws.js'
 import chokidar from 'chokidar'
 import { red } from 'kolorist'
 
-import { friendlyErrors, isCustomElement, md5, gzip } from './utils.js'
+import { friendlyErrors, defaultCustomElement, md5, gzip } from './utils.js'
 
 import { compileScss, parseJs, compileVue, parseHtml } from './compile-vue.js'
 
@@ -33,6 +33,7 @@ export default async function createServer(root = '', conf = {}) {
   const INJECT_SCSS = readFile(conf.inject?.scss)
   const LEGACY_MODE = !!conf.legacy
   const ENABLE_GZIP = !!conf.devServer.gzip
+  const { isCustomElement = defaultCustomElement } = conf.compileOptions || {}
 
   if (conf.imports['vue-dev']) {
     conf.imports.vue = conf.imports['vue-dev']
@@ -215,7 +216,7 @@ export default async function createServer(root = '', conf = {}) {
                 DEPLOY_PATH,
                 INJECT_SCSS,
                 LEGACY_MODE,
-                isCustomElement: conf.isCustomElement || isCustomElement
+                isCustomElement
               })
 
               res.setHeader('content-type', MIME_TYPES.js)
@@ -395,7 +396,7 @@ export default async function createServer(root = '', conf = {}) {
                       DEPLOY_PATH,
                       INJECT_SCSS,
                       LEGACY_MODE,
-                      isCustomElement: conf.isCustomElement || isCustomElement
+                      isCustomElement
                     })
                     let tmp = CACHE[filePath]
                     if (tmp.changed) {

lib/prod.js

@@ -1,33 +1,83 @@
-import { join, dirname, parse } from 'node:path'
+import { join, dirname, parse, normalize } from 'node:path'
+import { Worker, parentPort } from 'node:worker_threads'
+import os from 'node:os'
 import fs from 'iofs'
-import Es from 'esbuild'
-import { compileScss, parseJs, compileVue, parseHtml } from './compile-vue.js'
-import { isCustomElement } from './utils.js'
 
 const IS_WIN = process.platform === 'win32'
 const PREFIX = IS_WIN ? 'pages\\' : 'pages/'
+// 线程太多, 效率反而不高
+const THREADS_NUM = os.cpus().length > 4 ? 4 : os.cpus().length - 1
+const __filename = normalize(import.meta.url.slice(IS_WIN ? 8 : 7))
+const __dirname = dirname(__filename)
+const WORKER_POOL = new Set() // 线程池
+const JOBS_QUEUE = [] // 任务队列
 
 function readFile(file) {
   return (file && fs.cat(file)?.toString()) || ''
 }
 
+function doJob() {
+  while (JOBS_QUEUE.length && WORKER_POOL.size) {
+    let job = JOBS_QUEUE.shift()
+    let worker = WORKER_POOL.values().next().value
+
+    WORKER_POOL.delete(worker)
+
+    worker.once('message', _ => {
+      if (JOBS_QUEUE.length) {
+        WORKER_POOL.add(worker)
+        doJob()
+      } else {
+        worker.terminate()
+      }
+    })
+
+    worker.postMessage(job)
+  }
+}
+
 export default function compile(root = '', dist = '', conf = {}, verbose) {
   //
   const SOURCE_DIR = join(root, 'src')
   const PUBLIC_DIR = join(root, 'public')
   const DEPLOY_PATH = conf.base || '' // 部署目录, 默认是根目录部署
-  const IS_MPA = Object.keys(conf.pages).length > 1
+  const PAGES_KEYS = Object.keys(conf.pages)
-  const PAGES_PREFIX = Object.keys(conf.pages).map(it =>
+  const IS_MPA = PAGES_KEYS.length > 1
+  const PAGES_PREFIX = PAGES_KEYS.map(it =>
     IS_WIN ? `${PREFIX + it}\\` : `${PREFIX + it}/`
   )
   const INJECT_SCSS = readFile(conf.inject?.scss)
   const LEGACY_MODE = !!conf.legacy
+  const { isCustomElement } = conf.compileOptions || {}
 
   conf.inject = conf.inject || { scss: '' }
 
   let timeStart = Date.now()
   let template = fs.cat(join(process.cwd(), 'index.html')).toString()
-  let list = {}
+  let list = new Map()
+  let options = {
+    IS_MPA,
+    SOURCE_DIR,
+    DEPLOY_PATH,
+    INJECT_SCSS,
+    LEGACY_MODE,
+    // 线程通讯无法传递函数类型, 需要转为字符串, 之后再转回来
+    isCustomElement: isCustomElement ? isCustomElement.toString() : null
+  }
+
+  // 创建线程池
+  for (let i = 0; i < THREADS_NUM; i++) {
+    WORKER_POOL.add(
+      new Worker(join(__dirname, './thread.js'), {
+        workerData: {
+          options,
+          verbose,
+          dist,
+          imports: conf.imports
+        }
+      })
+    )
+  }
 
   fs.ls(SOURCE_DIR, true).forEach(path => {
     if (fs.isdir(path)) {
@@ -36,136 +86,46 @@ export default function compile(root = '', dist = '', conf = {}, verbose) {
 
     let name = path.slice(SOURCE_DIR.length + 1)
     let it = {
-      path,
       name,
       ext: parse(name).ext
     }
-    if (it.ext !== '') {
+    if (it.ext) {
       if (IS_MPA && it.name.startsWith(PREFIX)) {
         if (PAGES_PREFIX.some(it => it.startsWith(it.name))) {
-          return (list[path] = it)
+          list.set(path, it)
-        } else {
-          return
         }
+        return
       }
 
       if (it.path === conf.inject.scss) {
         return
       }
 
-      list[path] = it
+      list.set(path, it)
     }
   })
 
-  let compileFiles = function (currentPage, page, files) {
-    let options = {
-      IS_MPA,
-      currentPage,
-      SOURCE_DIR,
-      DEPLOY_PATH,
-      INJECT_SCSS,
-      LEGACY_MODE,
-      isCustomElement: conf.isCustomElement || isCustomElement
-    }
-
-    for (let k in files) {
-      let it = files[k]
-      // 入口文件, 特殊处理
-      if (page && it.path === page.entry) {
-        let entry = fs.cat(page.entry).toString()
-
-        entry = parseJs(entry, conf.imports, { ...options, IS_ENTRY: true })
-
-        let code = parseHtml(template, {
-          page,
-          imports: conf.imports,
-          entry,
-          LEGACY_MODE
-        })
-
-        fs.echo(code, join(dist, `${currentPage}.html`))
-        continue
-      }
-
-      verbose && console.log('  解析 %s ...', it.name)
-
-      let pageDir = IS_MPA && currentPage ? `pages/${currentPage}` : ''
-
-      switch (it.ext) {
-        case '.vue':
-          {
-            let code = compileVue(it.path, conf.imports, options)
-
-            Es.transform(code, { minify: true }).then(r => {
-              fs.echo(
-                r.code,
-                join(dist, 'assets/', pageDir, it.name.replace(/\.vue$/, '.js'))
-              )
-            })
-          }
-          break
-
-        case '.js':
-          {
-            let code = fs.cat(it.path)
-
-            code = parseJs(code + '', conf.imports, options)
-
-            Es.transform(code, { minify: true }).then(r => {
-              fs.echo(r.code, join(dist, 'assets/', pageDir, it.name))
-            })
-          }
-          break
-
-        case '.scss':
-        case '.css':
-          {
-            let target = join(
-              dist,
-              'assets/',
-              it.name.replace(/\.scss$/, '.css')
-            )
-            if (it.ext === '.css') {
-              fs.cp(it.path, target)
-            } else {
-              let code = compileScss(it.path)
-              fs.echo(code, target)
-            }
-          }
-          break
-
-        default:
-          fs.cp(it.path, join(dist, it.name))
-          break
-      }
-    }
-  }
-
   // 优先处理静态目录, 之后的源码目录中, 以便如果有产生相同的文件名, 则覆盖静态目录中的文件
   if (fs.isdir(PUBLIC_DIR)) {
     console.log('\n正在处理静态资源 ...')
     fs.ls(PUBLIC_DIR, true).forEach(it => {
       let ext = parse(it).ext
 
-      if (ext === '') {
+      if (ext && fs.isfile(it)) {
-        return
-      }
-
-      if (fs.isfile(it)) {
         let name = it.slice(PUBLIC_DIR.length + 1)
         verbose && console.log('  复制 %s ...', name)
         fs.cp(it, join(dist, name))
       }
     })
   }
-  console.log('')
 
-  for (let currentPage in conf.pages) {
+  if (IS_MPA) {
+    for (let currentPage of PAGES_KEYS) {
       let page = conf.pages[currentPage]
       let dir = dirname(page.entry)
-    let files = list
+      let files = new Map()
-    if (IS_MPA) {
+      let chunk = new Map()
-      files = {}
+
       fs.ls(dir, true).forEach(path => {
         if (fs.isdir(path)) {
           return
@@ -178,17 +138,42 @@ export default function compile(root = '', dist = '', conf = {}, verbose) {
           return
         }
 
-        delete list[path]
+        list.delete(path)
-        files[path] = { name, path, ext }
+        files.set(path, { name, ext })
       })
-    }
+      chunk.set(currentPage, { page, files })
-    console.log('正在生成 %s ...', `${currentPage}.html`)
+      JOBS_QUEUE.push(chunk)
-    compileFiles(currentPage, page, files)
+      doJob()
     }
 
-  if (IS_MPA) {
+    // 公共依赖
-    console.log('\n正在解析公共依赖 ...')
+    {
-    compileFiles('', null, list)
+      let chunk = new Map()
+      chunk.set('', { page: null, files: list })
+
+      JOBS_QUEUE.push(chunk)
+      doJob()
+    }
+  } else {
+    // 每个线程处理的文件数
+    let chunkSize = Math.ceil(list.size / THREADS_NUM)
+    let currentPage = PAGES_KEYS[0]
+    let page = conf.pages[currentPage]
+
+    list = [...list]
+
+    console.log(`正在生成 ${currentPage}.html ...`)
+
+    for (let i = 0; i < THREADS_NUM; i++) {
+      let start = i * chunkSize
+      let end = start + chunkSize
+      let chunk = new Map()
+
+      chunk.set(currentPage, { page, files: list.slice(start, end) })
+
+      JOBS_QUEUE.push(chunk)
+      doJob()
+    }
   }
 
   process.on('exit', _ => {

lib/thread.js

@@ -0,0 +1,34 @@
+/**
+ * 子线程
+ * @author yutent<yutent.io@gmail.com>
+ * @date 2023/06/14 16:15:39
+ */
+import { parentPort, workerData } from 'node:worker_threads'
+import { compileFiles } from './compile.js'
+import { defaultCustomElement } from './utils.js'
+
+const { options, verbose, dist, imports } = workerData
+
+options.isCustomElement = options.isCustomElement
+  ? Function('return ' + options.isCustomElement)()
+  : defaultCustomElement
+
+async function doJob(job) {
+  let [currentPage, { page, files }] = job.entries().next().value
+
+  options.IS_MPA &&
+    console.log(
+      currentPage
+        ? `正在生成 ${currentPage}.html ...`
+        : '\n正在解析公共依赖 ...'
+    )
+
+  await compileFiles(currentPage, page, files, options, {
+    verbose,
+    dist,
+    imports
+  })
+  parentPort.postMessage('ok')
+}
+
+parentPort.on('message', doJob)

lib/utils.js

@@ -7,7 +7,6 @@
 import { createHash, randomUUID } from 'node:crypto'
 import { join } from 'node:path'
 import { gzipSync } from 'node:zlib'
-import { Worker } from 'node:worker_threads'
 import { red, cyan, blue } from 'kolorist'
 
 // 修正路径合并 避免在windows下被转义
@@ -95,19 +94,7 @@ export function createHmrScript(legacy, session = '') {
   `
 }
 
-export function isCustomElement(tag) {
+// 默认的 web components 判断
+export function defaultCustomElement(tag) {
   return tag.startsWith('wc-')
 }
-
-export function startWorker(file) {
-  return new Promise((resolve, reject) => {
-    let worker = new Worker(file)
-    worker.on('message', resolve)
-    worker.on('error', reject)
-    worker.on('exit', code => {
-      if (code !== 0) {
-        reject(new Error(`Worker stopped with exit code ${code}`))
-      }
-    })
-  })
-}

lib/ws.js

@@ -32,7 +32,7 @@ class WebSocket {
 
   send(msg = {}) {
     if (this.#clients.size) {
-      for (let [key, client] of this.#clients.entries()) {
+      for (let [key, client] of this.#clients) {
         client.send(JSON.stringify(msg))
       }
     } else {