chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,41 @@
|
||||
/* eslint-disable no-console*/
|
||||
import corpus from 'nlp-corpus'
|
||||
import nlp from '../../../src/three.js'
|
||||
import plugin from '../src/plugin.js'
|
||||
nlp.extend(plugin)
|
||||
|
||||
const n = 30000
|
||||
console.log(` -- processing ${n.toLocaleString()} sentences-`)
|
||||
const docs = corpus.all().slice(30000, n + 30000)
|
||||
|
||||
const percent = (part, total) => {
|
||||
let num = (part / total) * 100;
|
||||
num = Math.round(num * 10) / 10;
|
||||
return num;
|
||||
};
|
||||
|
||||
let found = {}
|
||||
let matches = 0
|
||||
|
||||
docs.forEach(str => {
|
||||
const doc = nlp(str)
|
||||
const res = doc.dates().json({ normal: true })
|
||||
res.forEach(o => {
|
||||
console.log(o.normal)
|
||||
found[o.normal] = found[o.normal] || 0
|
||||
found[o.normal] += 1
|
||||
matches += 1
|
||||
if (o.normal === 'at some' || o.normal === "or" || o.normal === "but" || o.normal === "to") {
|
||||
console.log(' =-=-=-=-\n\n ', str)
|
||||
}
|
||||
})
|
||||
})
|
||||
// sort by freq
|
||||
found = Object.keys(found).map(k => [k, found[k]])
|
||||
found = found.sort((a, b) => (a[1] > b[1] ? -1 : 0))
|
||||
found = found.slice(0, 400)
|
||||
|
||||
|
||||
console.log(JSON.stringify(found, null, 2))
|
||||
console.log(`found ${matches.toLocaleString()} matches in ${n} sentences`)
|
||||
console.log(percent(matches, n) + '%')
|
||||
@@ -0,0 +1,6 @@
|
||||
import fs from 'fs'
|
||||
// avoid requiring our whole package.json file
|
||||
// make a small file for our version number
|
||||
const pkg = JSON.parse(fs.readFileSync('./package.json').toString())
|
||||
|
||||
fs.writeFileSync('./src/_version.js', `export default '${pkg.version}'`)
|
||||
Reference in New Issue
Block a user