wehub-resource-sync
28 lines
758 B
JavaScript
28 lines
758 B
JavaScript
/* eslint-disable no-console */
|
|
import corpus from 'nlp-corpus'
|
|
import fs from 'fs'
|
|
import nlp from '../../../src/three.js'
|
|
import plugin from '../src/plugin.js'
|
|
nlp.extend(plugin)
|
|
import idf from '../src/tfidf/idf.js'
|
|
import zipUp from './pack.js'
|
|
|
|
const fileSize = (pathStr) => {
|
|
const kb = fs.statSync(pathStr).size / 1024
|
|
return Math.round(kb) + 'kb'
|
|
}
|
|
|
|
// let txt = corpus.some(1000).join('\n')
|
|
const txt = corpus.all().join(`\n`)
|
|
|
|
const doc = nlp(txt).compute('root')
|
|
const counts = idf(doc, { use: 'root', min: 4 })
|
|
|
|
// collect by freq
|
|
const byFreq = zipUp(counts)
|
|
|
|
// console.log(counts)
|
|
const out = "export default " + JSON.stringify(byFreq, null, 2)
|
|
fs.writeFileSync('./src/tfidf/_model.js', out)
|
|
console.log(fileSize('./src/tfidf/_model.js'))
|