wehub-resource-sync
7254f7b4d1
Build / Build (macos-latest) (push) Has been cancelled
Build / Build (ubuntu-latest) (push) Has been cancelled
Build / Build (windows-latest) (push) Has been cancelled
Build / Analyze (javascript) (push) Has been cancelled
Build / Analyze (python) (push) Has been cancelled
2338 lines
77 KiB
JavaScript
2338 lines
77 KiB
JavaScript
|
|
// Experimental
|
|
|
|
import * as base from './base.js';
|
|
|
|
const mlnet = {};
|
|
|
|
mlnet.ModelFactory = class {
|
|
|
|
async match(context) {
|
|
const entries = await context.peek('zip');
|
|
if (entries instanceof Map && entries.size > 0) {
|
|
const root = new Set(['TransformerChain', 'Predictor']);
|
|
if (Array.from(entries.keys()).some((name) => root.has(name.split('\\').shift().split('/').shift()))) {
|
|
return context.set('mlnet', entries);
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async open(context) {
|
|
const metadata = await context.metadata('mlnet-metadata.json');
|
|
const reader = new mlnet.ModelReader(context.value);
|
|
await reader.resolve(context);
|
|
return new mlnet.Model(metadata, reader);
|
|
}
|
|
};
|
|
|
|
mlnet.Model = class {
|
|
|
|
constructor(metadata, reader) {
|
|
this.format = "ML.NET";
|
|
if (reader.version && reader.version.length > 0) {
|
|
this.format += ` v${reader.version}`;
|
|
}
|
|
this.modules = [new mlnet.Module(metadata, reader)];
|
|
}
|
|
};
|
|
|
|
mlnet.Module = class {
|
|
|
|
constructor(metadata, reader) {
|
|
this.inputs = [];
|
|
this.outputs = [];
|
|
this.nodes = [];
|
|
this.groups = false;
|
|
const values = new Map();
|
|
values.map = (name, type) => {
|
|
if (!values.has(name)) {
|
|
values.set(name, new mlnet.Value(name, type || null));
|
|
} else if (type) {
|
|
throw new mlnet.Error(`Duplicate value '${name}'.`);
|
|
}
|
|
return values.get(name);
|
|
};
|
|
if (reader.schema && reader.schema.inputs) {
|
|
for (const input of reader.schema.inputs) {
|
|
const argument = new mlnet.Argument(input.name, [values.map(input.name, new mlnet.TensorType(input.type))]);
|
|
this.inputs.push(argument);
|
|
}
|
|
}
|
|
const createNode = (scope, group, transformer) => {
|
|
if (transformer.inputs && transformer.outputs) {
|
|
for (const input of transformer.inputs) {
|
|
input.name = scope[input.name] ? scope[input.name].argument : input.name;
|
|
}
|
|
for (const output of transformer.outputs) {
|
|
if (scope[output.name]) {
|
|
scope[output.name].counter++;
|
|
const next = `${output.name}\n${scope[output.name].counter}`; // custom argument id
|
|
scope[output.name].argument = next;
|
|
output.name = next;
|
|
} else {
|
|
scope[output.name] = {
|
|
argument: output.name,
|
|
counter: 0
|
|
};
|
|
}
|
|
}
|
|
}
|
|
const node = new mlnet.Node(metadata, transformer, group, values);
|
|
this.nodes.push(node);
|
|
};
|
|
/* eslint-disable no-use-before-define */
|
|
const loadChain = (scope, name, chain) => {
|
|
this.groups = true;
|
|
const group = name.split('/').splice(1).join('/');
|
|
for (const childTransformer of chain) {
|
|
loadTransformer(scope, group, childTransformer);
|
|
}
|
|
};
|
|
const loadTransformer = (scope, group, transformer) => {
|
|
switch (transformer.__type__) {
|
|
case 'TransformerChain':
|
|
case 'Text':
|
|
loadChain(scope, transformer.__name__, transformer.chain);
|
|
break;
|
|
default:
|
|
createNode(scope, group, transformer);
|
|
break;
|
|
}
|
|
};
|
|
const scope = new Map();
|
|
if (reader.schema && reader.schema.inputs) {
|
|
for (const input of reader.schema.inputs) {
|
|
scope[input.name] = { argument: input.name, counter: 0 };
|
|
}
|
|
}
|
|
if (reader.dataLoaderModel) {
|
|
loadTransformer(scope, '', reader.dataLoaderModel);
|
|
}
|
|
if (reader.predictor) {
|
|
loadTransformer(scope, '', reader.predictor);
|
|
}
|
|
if (reader.transformerChain) {
|
|
loadTransformer(scope, '', reader.transformerChain);
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.Argument = class {
|
|
|
|
constructor(name, value, type = null) {
|
|
this.name = name;
|
|
this.value = value;
|
|
this.type = type;
|
|
}
|
|
};
|
|
|
|
mlnet.Value = class {
|
|
|
|
constructor(name, type) {
|
|
if (typeof name !== 'string') {
|
|
throw new mlnet.Error(`Invalid value identifier '${JSON.stringify(name)}'.`);
|
|
}
|
|
this.name = name;
|
|
this.type = type;
|
|
this.initializer = null;
|
|
}
|
|
};
|
|
|
|
mlnet.Node = class {
|
|
|
|
constructor(metadata, obj, group, values) {
|
|
const op = obj.__type__;
|
|
this.type = metadata.type(op) || { name: op || '?' };
|
|
this.name = obj.__name__ || '';
|
|
this.group = group || '';
|
|
this.inputs = [];
|
|
this.outputs = [];
|
|
this.attributes = [];
|
|
if (values && obj.inputs) {
|
|
for (let i = 0; i < obj.inputs.length; i++) {
|
|
const value = values.map(obj.inputs[i].name);
|
|
this.inputs.push(new mlnet.Argument(i.toString(), [value]));
|
|
}
|
|
}
|
|
if (values && obj.outputs) {
|
|
for (let i = 0; i < obj.outputs.length; i++) {
|
|
this.outputs.push(new mlnet.Argument(i.toString(), [values.map(obj.outputs[i].name)]));
|
|
}
|
|
}
|
|
for (const [name, raw] of Object.entries(obj).filter(([key]) => !key.startsWith('_') && key !== 'inputs' && key !== 'outputs')) {
|
|
const schema = metadata.attribute(op, name);
|
|
let value = raw;
|
|
let type = null;
|
|
if (schema) {
|
|
type = schema.type ? schema.type : null;
|
|
value = mlnet.Utility.enum(type, value);
|
|
}
|
|
if (value && typeof value === 'object' && !Array.isArray(value) && Array.isArray(value.nodes)) {
|
|
type = 'graph';
|
|
} else if (value && typeof value === 'object' && !Array.isArray(value) && value.__type__) {
|
|
value = new mlnet.Node(metadata, value);
|
|
type = 'object';
|
|
} else if (Array.isArray(value) && value.length > 0 && value.every((item) => item && item.__type__)) {
|
|
value = value.map((item) => new mlnet.Node(metadata, item));
|
|
type = 'object[]';
|
|
}
|
|
this.attributes.push(new mlnet.Argument(name, value, type));
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.TensorType = class {
|
|
|
|
constructor(codec) {
|
|
mlnet.TensorType._map = mlnet.TensorType._map || new Map([
|
|
['Byte', 'uint8'],
|
|
['Boolean', 'boolean'],
|
|
['Single', 'float32'],
|
|
['Double', 'float64'],
|
|
['UInt32', 'uint32'],
|
|
['Int32', 'int32'],
|
|
['Int64', 'int64'],
|
|
['TextSpan', 'string']
|
|
]);
|
|
this.dataType = '?';
|
|
this.shape = new mlnet.TensorShape(null);
|
|
if (mlnet.TensorType._map.has(codec.name)) {
|
|
this.dataType = mlnet.TensorType._map.get(codec.name);
|
|
} else if (codec.name === 'VBuffer') {
|
|
if (mlnet.TensorType._map.has(codec.itemType.name)) {
|
|
this.dataType = mlnet.TensorType._map.get(codec.itemType.name);
|
|
} else {
|
|
throw new mlnet.Error(`Unsupported data type '${codec.itemType.name}'.`);
|
|
}
|
|
this.shape = new mlnet.TensorShape(codec.dims);
|
|
} else if (codec.name === 'Key2') {
|
|
this.dataType = 'key2';
|
|
} else {
|
|
throw new mlnet.Error(`Unsupported data type '${codec.name}'.`);
|
|
}
|
|
}
|
|
|
|
toString() {
|
|
return this.dataType + this.shape.toString();
|
|
}
|
|
};
|
|
|
|
mlnet.TensorShape = class {
|
|
|
|
constructor(dimensions) {
|
|
this.dimensions = dimensions;
|
|
}
|
|
|
|
toString() {
|
|
if (!this.dimensions || this.dimensions.length === 0) {
|
|
return '';
|
|
}
|
|
return `[${this.dimensions.join(',')}]`;
|
|
}
|
|
};
|
|
|
|
mlnet.ModelReader = class {
|
|
|
|
constructor(entries) {
|
|
const catalog = new mlnet.ComponentCatalog();
|
|
catalog.register('AffineNormExec', mlnet.AffineNormSerializationUtils);
|
|
catalog.register('AnomalyPredXfer', mlnet.AnomalyPredictionTransformer);
|
|
catalog.register('BinaryPredXfer', mlnet.BinaryPredictionTransformer);
|
|
catalog.register('BinaryLoader', mlnet.BinaryLoader);
|
|
catalog.register('CaliPredExec', mlnet.CalibratedPredictor);
|
|
catalog.register('CdfNormalizeFunction', mlnet.CdfColumnFunction);
|
|
catalog.register('CharToken', mlnet.TokenizingByCharactersTransformer);
|
|
catalog.register('ChooseColumnsTransform', mlnet.ColumnSelectingTransformer);
|
|
catalog.register('ClusteringPredXfer', mlnet.ClusteringPredictionTransformer);
|
|
catalog.register('ConcatTransform', mlnet.ColumnConcatenatingTransformer);
|
|
catalog.register('CopyTransform', mlnet.ColumnCopyingTransformer);
|
|
catalog.register('ConvertTransform', mlnet.TypeConvertingTransformer);
|
|
catalog.register('CSharpTransform', mlnet.CSharpTransform);
|
|
catalog.register('DropColumnsTransform', mlnet.DropColumnsTransform);
|
|
catalog.register('FAFMPredXfer', mlnet.FieldAwareFactorizationMachinePredictionTransformer);
|
|
catalog.register('FastForestBinaryExec', mlnet.FastForestClassificationPredictor);
|
|
catalog.register('FastTreeBinaryExec', mlnet.FastTreeBinaryModelParameters);
|
|
catalog.register('FastTreeTweedieExec', mlnet.FastTreeTweedieModelParameters);
|
|
catalog.register('FastTreeRankerExec', mlnet.FastTreeRankingModelParameters);
|
|
catalog.register('FastTreeRegressionExec', mlnet.FastTreeRegressionModelParameters);
|
|
catalog.register('FeatWCaliPredExec', mlnet.FeatureWeightsCalibratedModelParameters);
|
|
catalog.register('FieldAwareFactMacPredict', mlnet.FieldAwareFactorizationMachineModelParameters);
|
|
catalog.register('GcnTransform', mlnet.LpNormNormalizingTransformer);
|
|
catalog.register('GenericScoreTransform', mlnet.GenericScoreTransform);
|
|
catalog.register('IidChangePointDetector', mlnet.IidChangePointDetector);
|
|
catalog.register('IidSpikeDetector', mlnet.IidSpikeDetector);
|
|
catalog.register('ImageClassificationTrans', mlnet.ImageClassificationTransformer);
|
|
catalog.register('ImageClassificationPred', mlnet.ImageClassificationModelParameters);
|
|
catalog.register('ImageLoaderTransform', mlnet.ImageLoadingTransformer);
|
|
catalog.register('ImageScalerTransform', mlnet.ImageResizingTransformer);
|
|
catalog.register('ImagePixelExtractor', mlnet.ImagePixelExtractingTransformer);
|
|
catalog.register('KeyToValueTransform', mlnet.KeyToValueMappingTransformer);
|
|
catalog.register('KeyToVectorTransform', mlnet.KeyToVectorMappingTransformer);
|
|
catalog.register('KMeansPredictor', mlnet.KMeansModelParameters);
|
|
catalog.register('LinearRegressionExec', mlnet.LinearRegressionModelParameters);
|
|
catalog.register('LightGBMRegressionExec', mlnet.LightGbmRegressionModelParameters);
|
|
catalog.register('LightGBMBinaryExec', mlnet.LightGbmBinaryModelParameters);
|
|
catalog.register('Linear2CExec', mlnet.LinearBinaryModelParameters);
|
|
catalog.register('LinearModelStats', mlnet.LinearModelParameterStatistics);
|
|
catalog.register('MaFactPredXf', mlnet.MatrixFactorizationPredictionTransformer);
|
|
catalog.register('MFPredictor', mlnet.MatrixFactorizationModelParameters);
|
|
catalog.register('MulticlassLinear', mlnet.LinearMulticlassModelParameters);
|
|
catalog.register('MultiClassLRExec', mlnet.MaximumEntropyModelParameters);
|
|
catalog.register('MultiClassNaiveBayesPred', mlnet.NaiveBayesMulticlassModelParameters);
|
|
catalog.register('MultiClassNetPredictor', mlnet.MultiClassNetPredictor);
|
|
catalog.register('MulticlassPredXfer', mlnet.MulticlassPredictionTransformer);
|
|
catalog.register('NAReplaceTransform', mlnet.MissingValueReplacingTransformer);
|
|
catalog.register('NgramTransform', mlnet.NgramExtractingTransformer);
|
|
catalog.register('NgramHashTransform', mlnet.NgramHashingTransformer);
|
|
catalog.register('NltTokenizeTransform', mlnet.NltTokenizeTransform);
|
|
catalog.register('Normalizer', mlnet.NormalizingTransformer);
|
|
catalog.register('NormalizeTransform', mlnet.NormalizeTransform);
|
|
catalog.register('OnnxTransform', mlnet.OnnxTransformer);
|
|
catalog.register('OptColTransform', mlnet.OptionalColumnTransform);
|
|
catalog.register('OVAExec', mlnet.OneVersusAllModelParameters);
|
|
catalog.register('pcaAnomExec', mlnet.PcaModelParameters);
|
|
catalog.register('PcaTransform', mlnet.PrincipalComponentAnalysisTransformer);
|
|
catalog.register('PipeDataLoader', mlnet.CompositeDataLoader);
|
|
catalog.register('PlattCaliExec', mlnet.PlattCalibrator);
|
|
catalog.register('PMixCaliPredExec', mlnet.ParameterMixingCalibratedModelParameters);
|
|
catalog.register('PoissonRegressionExec', mlnet.PoissonRegressionModelParameters);
|
|
catalog.register('ProtonNNMCPred', mlnet.ProtonNNMCPred);
|
|
catalog.register('RegressionPredXfer', mlnet.RegressionPredictionTransformer);
|
|
catalog.register('RowToRowMapper', mlnet.RowToRowMapperTransform);
|
|
catalog.register('SsaForecasting', mlnet.SsaForecastingTransformer);
|
|
catalog.register('SSAModel', mlnet.AdaptiveSingularSpectrumSequenceModelerInternal);
|
|
catalog.register('SelectColumnsTransform', mlnet.ColumnSelectingTransformer);
|
|
catalog.register('StopWordsTransform', mlnet.StopWordsTransform);
|
|
catalog.register('TensorFlowTransform', mlnet.TensorFlowTransformer);
|
|
catalog.register('TermLookupTransform', mlnet.ValueMappingTransformer);
|
|
catalog.register('TermTransform', mlnet.ValueToKeyMappingTransformer);
|
|
catalog.register('TermManager', mlnet.TermManager);
|
|
catalog.register('Text', mlnet.TextFeaturizingEstimator);
|
|
catalog.register('TextLoader', mlnet.TextLoader);
|
|
catalog.register('TextNormalizerTransform', mlnet.TextNormalizingTransformer);
|
|
catalog.register('TokenizeTextTransform', mlnet.WordTokenizingTransformer);
|
|
catalog.register('TransformerChain', mlnet.TransformerChain);
|
|
catalog.register('ValueMappingTransformer', mlnet.ValueMappingTransformer);
|
|
catalog.register('XGBoostMulticlass', mlnet.XGBoostMulticlass);
|
|
this._resolve = [];
|
|
const root = new mlnet.ModelHeader(catalog, entries, '', null, this._resolve);
|
|
const version = root.openText('TrainingInfo/Version.txt');
|
|
if (version) {
|
|
[this.version] = version.split(/[\s+\r]+/);
|
|
}
|
|
const schemaReader = root.openBinary('Schema');
|
|
if (schemaReader) {
|
|
this.schema = new mlnet.BinaryLoader(null, schemaReader).schema;
|
|
}
|
|
const transformerChain = root.open('TransformerChain');
|
|
if (transformerChain) {
|
|
this.transformerChain = transformerChain;
|
|
}
|
|
const dataLoaderModel = root.open('DataLoaderModel');
|
|
if (dataLoaderModel) {
|
|
this.dataLoaderModel = dataLoaderModel;
|
|
}
|
|
const predictor = root.open('Predictor');
|
|
if (predictor) {
|
|
this.predictor = predictor;
|
|
}
|
|
}
|
|
|
|
async resolve(context) {
|
|
const resolve = async (entry) => {
|
|
let module = null;
|
|
let content = '';
|
|
if (entry.format === 'tf') {
|
|
module = await context.require('./tf');
|
|
content = context.context('model.pb', entry.bytes);
|
|
} else if (entry.format === 'onnx') {
|
|
module = await context.require('./onnx');
|
|
content = context.context('model.onnx', entry.bytes);
|
|
} else {
|
|
throw new mlnet.Error(`Unsupported ML.NET model format '${entry.format}'.`);
|
|
}
|
|
const factory = new module.ModelFactory();
|
|
await factory.match(content);
|
|
const model = await factory.open(content);
|
|
if (model && Array.isArray(model.modules) && model.modules.length > 0) {
|
|
return model.modules[0];
|
|
}
|
|
return null;
|
|
};
|
|
const results = await Promise.all(this._resolve.map((entry) => resolve(entry).catch(() => null)));
|
|
for (let i = 0; i < this._resolve.length; i++) {
|
|
if (results[i]) {
|
|
this._resolve[i].target.Model = results[i];
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.ComponentCatalog = class {
|
|
|
|
constructor() {
|
|
this._registry = new Map();
|
|
}
|
|
|
|
register(signature, type) {
|
|
this._registry.set(signature, type);
|
|
}
|
|
|
|
create(signature, context) {
|
|
if (!this._registry.has(signature)) {
|
|
throw new mlnet.Error(`Unsupported loader signature '${signature}'.`);
|
|
}
|
|
const type = this._registry.get(signature);
|
|
return Reflect.construct(type, [context]);
|
|
}
|
|
};
|
|
|
|
mlnet.ModelHeader = class {
|
|
|
|
constructor(catalog, entries, directory, data, resolve) {
|
|
this._entries = entries;
|
|
this._catalog = catalog;
|
|
this._directory = directory;
|
|
this._resolve = resolve;
|
|
if (data) {
|
|
const reader = new mlnet.BinaryReader(data);
|
|
const decoder = new TextDecoder('ascii');
|
|
reader.assert('ML\0MODEL');
|
|
this.versionWritten = reader.uint32();
|
|
this.versionReadable = reader.uint32();
|
|
const modelBlockOffset = reader.uint64().toNumber();
|
|
/* let modelBlockSize = */ reader.uint64();
|
|
const stringTableOffset = reader.uint64().toNumber();
|
|
const stringTableSize = reader.uint64().toNumber();
|
|
const stringCharsOffset = reader.uint64().toNumber();
|
|
/* v stringCharsSize = */ reader.uint64();
|
|
this.modelSignature = decoder.decode(reader.read(8));
|
|
this.modelVersionWritten = reader.uint32();
|
|
this.modelVersionReadable = reader.uint32();
|
|
this.loaderSignature = decoder.decode(reader.read(24).filter((c) => c !== 0));
|
|
this.loaderSignatureAlt = decoder.decode(reader.read(24).filter((c) => c !== 0));
|
|
const tailOffset = reader.uint64().toNumber();
|
|
/* let tailLimit = */ reader.uint64();
|
|
const assemblyNameOffset = reader.uint64().toNumber();
|
|
const assemblyNameSize = reader.uint32();
|
|
if (stringTableOffset !== 0 && stringCharsOffset !== 0) {
|
|
reader.seek(stringTableOffset);
|
|
const stringCount = stringTableSize >> 3;
|
|
const stringSizes = [];
|
|
let previousStringSize = 0;
|
|
for (let i = 0; i < stringCount; i++) {
|
|
const stringSize = reader.uint64().toNumber();
|
|
stringSizes.push(stringSize - previousStringSize);
|
|
previousStringSize = stringSize;
|
|
}
|
|
reader.seek(stringCharsOffset);
|
|
this.strings = [];
|
|
for (let i = 0; i < stringCount; i++) {
|
|
const cch = stringSizes[i] >> 1;
|
|
let sb = '';
|
|
for (let ich = 0; ich < cch; ich++) {
|
|
sb += String.fromCharCode(reader.uint16());
|
|
}
|
|
this.strings.push(sb);
|
|
}
|
|
}
|
|
if (assemblyNameOffset !== 0) {
|
|
reader.seek(assemblyNameOffset);
|
|
this.assemblyName = decoder.decode(reader.read(assemblyNameSize));
|
|
}
|
|
reader.seek(tailOffset);
|
|
reader.assert('LEDOM\0LM');
|
|
this._reader = reader;
|
|
this._reader.seek(modelBlockOffset);
|
|
}
|
|
}
|
|
|
|
get reader() {
|
|
return this._reader;
|
|
}
|
|
|
|
string(empty) {
|
|
const id = this.reader.int32();
|
|
if (empty === null && id < 0) {
|
|
return null;
|
|
}
|
|
return this.strings[id];
|
|
}
|
|
|
|
open(name) {
|
|
const dir = this._directory.length > 0 ? `${this._directory}/` : this._directory;
|
|
name = dir + name;
|
|
const key = `${name}/Model.key`;
|
|
const stream = this._entries.get(key) || this._entries.get(key.replace(/\//g, '\\'));
|
|
if (stream) {
|
|
const buffer = stream.peek();
|
|
const context = new mlnet.ModelHeader(this._catalog, this._entries, name, buffer, this._resolve);
|
|
const value = this._catalog.create(context.loaderSignature, context);
|
|
value.__type__ = value.__type__ || context.loaderSignature;
|
|
value.__name__ = name;
|
|
return value;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
openBinary(name) {
|
|
const dir = this._directory.length > 0 ? `${this._directory}/` : this._directory;
|
|
name = dir + name;
|
|
const stream = this._entries.get(name) || this._entries.get(name.replace(/\//g, '\\'));
|
|
if (stream) {
|
|
return new mlnet.BinaryReader(stream);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
openText(name) {
|
|
const dir = this._directory.length > 0 ? `${this._directory}/` : this._directory;
|
|
name = dir + name;
|
|
const stream = this._entries.get(name) || this._entries.get(name.replace(/\//g, '\\'));
|
|
if (stream) {
|
|
const buffer = stream.peek();
|
|
const decoder = new TextDecoder('utf-8');
|
|
return decoder.decode(buffer);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
check(signature, verWrittenCur, verWeCanReadBack) {
|
|
return signature === this.modelSignature && verWrittenCur >= this.modelVersionReadable && verWeCanReadBack <= this.modelVersionWritten;
|
|
}
|
|
|
|
resolve(target, format, bytes) {
|
|
this._resolve.push({ target, format, bytes });
|
|
}
|
|
};
|
|
|
|
mlnet.BinaryReader = class {
|
|
|
|
constructor(data) {
|
|
this._reader = base.BinaryReader.open(data);
|
|
}
|
|
|
|
seek(position) {
|
|
this._reader.seek(position);
|
|
}
|
|
|
|
skip(offset) {
|
|
this._reader.skip(offset);
|
|
}
|
|
|
|
read(length) {
|
|
return this._reader.read(length);
|
|
}
|
|
|
|
boolean() {
|
|
return this._reader.boolean();
|
|
}
|
|
|
|
booleans(count) {
|
|
const values = [];
|
|
for (let i = 0; i < count; i++) {
|
|
values.push(this.boolean());
|
|
}
|
|
return values;
|
|
}
|
|
|
|
byte() {
|
|
return this._reader.byte();
|
|
}
|
|
|
|
int16() {
|
|
return this._reader.int16();
|
|
}
|
|
|
|
int32() {
|
|
return this._reader.int32();
|
|
}
|
|
|
|
int32s(count) {
|
|
const values = [];
|
|
for (let i = 0; i < count; i++) {
|
|
values.push(this.int32());
|
|
}
|
|
return values;
|
|
}
|
|
|
|
int64() {
|
|
return this._reader.int64();
|
|
}
|
|
|
|
uint16() {
|
|
return this._reader.uint16();
|
|
}
|
|
|
|
uint32() {
|
|
return this._reader.uint32();
|
|
}
|
|
|
|
uint32s(count) {
|
|
const values = [];
|
|
for (let i = 0; i < count; i++) {
|
|
values.push(this.uint32());
|
|
}
|
|
return values;
|
|
}
|
|
|
|
uint64() {
|
|
return this._reader.uint64();
|
|
}
|
|
|
|
float32() {
|
|
return this._reader.float32();
|
|
}
|
|
|
|
float32s(count) {
|
|
const values = [];
|
|
for (let i = 0; i < count; i++) {
|
|
values.push(this.float32());
|
|
}
|
|
return values;
|
|
}
|
|
|
|
float64() {
|
|
return this._reader.float64();
|
|
}
|
|
|
|
float64s(count) {
|
|
const values = [];
|
|
for (let i = 0; i < count; i++) {
|
|
values.push(this.float64());
|
|
}
|
|
return values;
|
|
}
|
|
|
|
string() {
|
|
const size = this.leb128();
|
|
const buffer = this.read(size);
|
|
return new TextDecoder('utf-8').decode(buffer);
|
|
}
|
|
|
|
leb128() {
|
|
let result = 0;
|
|
let shift = 0;
|
|
let value = 0;
|
|
do {
|
|
value = this.byte();
|
|
result |= (value & 0x7F) << shift;
|
|
shift += 7;
|
|
} while ((value & 0x80) !== 0);
|
|
return result;
|
|
}
|
|
|
|
match(text) {
|
|
const position = this.position;
|
|
for (let i = 0; i < text.length; i++) {
|
|
if (this.byte() !== text.charCodeAt(i)) {
|
|
this.seek(position);
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
assert(text) {
|
|
if (!this.match(text)) {
|
|
throw new mlnet.Error(`Invalid '${text.split('\0').join('')}' signature.`);
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.BinaryLoader = class { // 'BINLOADR'
|
|
|
|
constructor(context, reader) {
|
|
if (context) {
|
|
if (context.modelVersionWritten >= 0x00010002) {
|
|
this.Threads = context.reader.int32();
|
|
this.GeneratedRowIndexName = context.string(null);
|
|
}
|
|
this.ShuffleBlocks = context.modelVersionWritten >= 0x00010003 ? context.reader.float64() : 4;
|
|
reader = context.openBinary('Schema.idv');
|
|
}
|
|
// https://github.com/dotnet/machinelearning/blob/master/docs/code/IdvFileFormat.md
|
|
reader.assert('CML\0DVB\0');
|
|
reader.skip(8); // version
|
|
reader.skip(8); // compatibleVersion
|
|
const tableOfContentsOffset = reader.uint64().toNumber();
|
|
const tailOffset = reader.int64().toNumber();
|
|
reader.int64(); // rowCount
|
|
const columnCount = reader.int32();
|
|
reader.seek(tailOffset);
|
|
reader.assert('\0BVD\0LMC');
|
|
reader.seek(tableOfContentsOffset);
|
|
this.schema = {};
|
|
this.schema.inputs = [];
|
|
const columns = new Map();
|
|
for (let c = 0; c < columnCount; c ++) {
|
|
const input = {};
|
|
input.name = reader.string();
|
|
input.type = new mlnet.Codec(reader);
|
|
input.compression = reader.byte(); // None = 0, Deflate = 1
|
|
input.rowsPerBlock = reader.leb128();
|
|
input.lookupOffset = reader.int64();
|
|
input.metadataTocOffset = reader.int64();
|
|
columns.set(input.name, input);
|
|
}
|
|
for (const input of columns.values()) {
|
|
this.schema.inputs.push(input);
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.TransformerChain = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
const length = reader.int32();
|
|
this.scopes = [];
|
|
this.chain = [];
|
|
for (let i = 0; i < length; i++) {
|
|
this.scopes.push(reader.int32()); // 0x01 = Training, 0x02 = Testing, 0x04 = Scoring
|
|
const dirName = `Transform_${(`00${i}`).slice(-3)}`;
|
|
const transformer = context.open(dirName);
|
|
this.chain.push(transformer);
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.TransformBase = class {
|
|
};
|
|
|
|
mlnet.RowToRowTransformBase = class extends mlnet.TransformBase {
|
|
};
|
|
|
|
mlnet.RowToRowTransformerBase = class {
|
|
};
|
|
|
|
mlnet.RowToRowMapperTransformBase = class extends mlnet.RowToRowTransformBase {
|
|
};
|
|
|
|
mlnet.OneToOneTransformerBase = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
const n = reader.int32();
|
|
this.inputs = [];
|
|
this.outputs = [];
|
|
for (let i = 0; i < n; i++) {
|
|
const output = context.string();
|
|
const input = context.string();
|
|
this.outputs.push({ name: output });
|
|
this.inputs.push({ name: input });
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.ColumnCopyingTransformer = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
const length = reader.uint32();
|
|
this.inputs = [];
|
|
this.outputs = [];
|
|
for (let i = 0; i < length; i++) {
|
|
this.outputs.push({ name: context.string() });
|
|
this.inputs.push({ name: context.string() });
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.ColumnConcatenatingTransformer = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
if (context.modelVersionReadable >= 0x00010003) {
|
|
const count = reader.int32();
|
|
for (let i = 0; i < count; i++) {
|
|
this.outputs = [];
|
|
this.outputs.push({ name: context.string() });
|
|
const n = reader.int32();
|
|
this.inputs = [];
|
|
for (let j = 0; j < n; j++) {
|
|
const input = {
|
|
name: context.string()
|
|
};
|
|
const alias = context.string(null);
|
|
if (alias) {
|
|
input.alias = alias;
|
|
}
|
|
this.inputs.push(input);
|
|
}
|
|
}
|
|
} else {
|
|
this.precision = reader.int32();
|
|
const n = reader.int32();
|
|
const names = [];
|
|
const inputs = [];
|
|
for (let i = 0; i < n; i++) {
|
|
names.push(context.string());
|
|
const numSources = reader.int32();
|
|
const input = [];
|
|
for (let j = 0; j < numSources; j++) {
|
|
input.push(context.string());
|
|
}
|
|
inputs.push(input);
|
|
}
|
|
const aliases = [];
|
|
if (context.modelVersionReadable >= 0x00010002) {
|
|
for (let i = 0; i < n; i++) {
|
|
/* let length = inputs[i].length; */
|
|
const alias = {};
|
|
aliases.push(alias);
|
|
if (context.modelVersionReadable >= 0x00010002) {
|
|
for (;;) {
|
|
const j = reader.int32();
|
|
if (j === -1) {
|
|
break;
|
|
}
|
|
alias[j] = context.string();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (n > 1) {
|
|
throw new mlnet.Error(`Unsupported ColumnConcatenatingTransformer name count '${n}'.`);
|
|
}
|
|
|
|
this.outputs = [];
|
|
for (let i = 0; i < n; i++) {
|
|
this.outputs.push({
|
|
name: names[i]
|
|
});
|
|
this.inputs = inputs[i];
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.PredictionTransformerBase = class {
|
|
|
|
constructor(context) {
|
|
this.Model = context.open('Model');
|
|
const trainSchemaReader = context.openBinary('TrainSchema');
|
|
if (trainSchemaReader) {
|
|
this.schema = new mlnet.BinaryLoader(null, trainSchemaReader).schema;
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.MatrixFactorizationModelParameters = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
this.NumberOfRows = reader.int32();
|
|
if (context.modelVersionWritten < 0x00010002) {
|
|
reader.uint64(); // mMin
|
|
}
|
|
this.NumberOfColumns = reader.int32();
|
|
if (context.modelVersionWritten < 0x00010002) {
|
|
reader.uint64(); // nMin
|
|
}
|
|
this.ApproximationRank = reader.int32();
|
|
|
|
this._leftFactorMatrix = reader.float32s(this.NumberOfRows * this.ApproximationRank);
|
|
this._rightFactorMatrix = reader.float32s(this.NumberOfColumns * this.ApproximationRank);
|
|
}
|
|
};
|
|
|
|
mlnet.MatrixFactorizationPredictionTransformer = class extends mlnet.PredictionTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
this.MatrixColumnIndexColumnName = context.string();
|
|
this.MatrixRowIndexColumnName = context.string();
|
|
}
|
|
};
|
|
|
|
mlnet.FieldAwareFactorizationMachinePredictionTransformer = class extends mlnet.PredictionTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.inputs = [];
|
|
for (let i = 0; i < this.FieldCount; i++) {
|
|
this.inputs.push({ name: context.string() });
|
|
}
|
|
this.Threshold = reader.float32();
|
|
this.ThresholdColumn = context.string();
|
|
this.outputs = [];
|
|
this.outputs.push({ name: 'Score' });
|
|
this.outputs.push({ name: 'Probability' });
|
|
this.outputs.push({ name: 'PredictedLabel' });
|
|
}
|
|
};
|
|
|
|
mlnet.SingleFeaturePredictionTransformerBase = class extends mlnet.PredictionTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const featureColumn = context.string(null);
|
|
this.inputs = [];
|
|
this.inputs.push({ name: featureColumn });
|
|
this.outputs = [];
|
|
}
|
|
};
|
|
|
|
mlnet.ClusteringPredictionTransformer = class extends mlnet.SingleFeaturePredictionTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
this.outputs.push({ name: 'Score' });
|
|
this.outputs.push({ name: 'PredictedLabel' });
|
|
}
|
|
};
|
|
|
|
mlnet.AnomalyPredictionTransformer = class extends mlnet.SingleFeaturePredictionTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.Threshold = reader.float32();
|
|
this.ThresholdColumn = context.string();
|
|
this.outputs.push({ name: 'Score' });
|
|
this.outputs.push({ name: 'PredictedLabel' });
|
|
}
|
|
};
|
|
|
|
mlnet.AffineNormSerializationUtils = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
/* cbFloat = */ reader.int32();
|
|
this.NumFeatures = reader.int32();
|
|
const morphCount = reader.int32();
|
|
if (morphCount === -1) {
|
|
this.ScalesSparse = reader.float32s(reader.int32());
|
|
this.OffsetsSparse = reader.float32s(reader.int32());
|
|
} else {
|
|
// debugger;
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.RegressionPredictionTransformer = class extends mlnet.SingleFeaturePredictionTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
this.outputs.push({ name: 'Score' });
|
|
}
|
|
};
|
|
|
|
mlnet.BinaryPredictionTransformer = class extends mlnet.SingleFeaturePredictionTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.Threshold = reader.float32();
|
|
this.ThresholdColumn = context.string();
|
|
this.outputs.push({ name: 'Score' });
|
|
this.outputs.push({ name: 'PredictedLabel' });
|
|
}
|
|
};
|
|
|
|
mlnet.MulticlassPredictionTransformer = class extends mlnet.SingleFeaturePredictionTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
this.TrainLabelColumn = context.string(null);
|
|
this.inputs.push({ name: this.TrainLabelColumn });
|
|
if (context.modelVersionWritten >= 0x00010002) {
|
|
const scoreColumn = context.string(null);
|
|
const predictedLabelColumn = context.string(null);
|
|
this.outputs.push({ name: scoreColumn || 'Score' });
|
|
this.outputs.push({ name: predictedLabelColumn || 'PredictedLabel' });
|
|
} else {
|
|
this.outputs.push({ name: 'Score' });
|
|
this.outputs.push({ name: 'PredictedLabel' });
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.MissingValueReplacingTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
for (let i = 0; i < this.inputs.length; i++) {
|
|
const codec = new mlnet.Codec(reader);
|
|
const count = reader.int32();
|
|
this.values = codec.read(reader, count);
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.PredictorBase = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
if (reader.int32() !== 4) {
|
|
throw new mlnet.Error('Invalid float type size.');
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.ModelParametersBase = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
const cbFloat = reader.int32();
|
|
if (cbFloat !== 4) {
|
|
throw new mlnet.Error('This file was saved by an incompatible version.');
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.ImageClassificationModelParameters = class extends mlnet.ModelParametersBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.classCount = reader.int32();
|
|
this.imagePreprocessorTensorInput = reader.string();
|
|
this.imagePreprocessorTensorOutput = reader.string();
|
|
this.graphInputTensor = reader.string();
|
|
this.graphOutputTensor = reader.string();
|
|
const modelReader = context.openBinary('TFModel');
|
|
if (modelReader) {
|
|
const size = modelReader.uint32();
|
|
context.resolve(this, 'tf', modelReader.read(size));
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.NaiveBayesMulticlassModelParameters = class extends mlnet.ModelParametersBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this._labelHistogram = reader.int32s(reader.int32());
|
|
this._featureCount = reader.int32();
|
|
this._featureHistogram = [];
|
|
for (let i = 0; i < this._labelHistogram.length; i++) {
|
|
if (this._labelHistogram[i] > 0) {
|
|
this._featureHistogram.push(reader.int32s(this._featureCount));
|
|
}
|
|
}
|
|
this._absentFeaturesLogProb = reader.float64s(this._labelHistogram.length);
|
|
}
|
|
};
|
|
|
|
mlnet.LinearModelParameters = class extends mlnet.ModelParametersBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.Bias = reader.float32();
|
|
/* let len = */ reader.int32();
|
|
this.Indices = reader.int32s(reader.int32());
|
|
this.Weights = reader.float32s(reader.int32());
|
|
}
|
|
};
|
|
|
|
mlnet.LinearBinaryModelParameters = class extends mlnet.LinearModelParameters {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
if (context.modelVersionWritten > 0x00020001) {
|
|
this.Statistics = context.open('ModelStats');
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.ModelStatisticsBase = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
this.ParametersCount = reader.int32();
|
|
this.TrainingExampleCount = reader.int64().toNumber();
|
|
this.Deviance = reader.float32();
|
|
this.NullDeviance = reader.float32();
|
|
|
|
}
|
|
};
|
|
|
|
mlnet.LinearModelParameterStatistics = class extends mlnet.ModelStatisticsBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
if (context.modelVersionWritten < 0x00010002) {
|
|
if (!reader.boolean()) {
|
|
return;
|
|
}
|
|
}
|
|
const stdErrorValues = reader.float32s(this.ParametersCount);
|
|
const length = reader.int32();
|
|
if (length === this.ParametersCount) {
|
|
this._coeffStdError = stdErrorValues;
|
|
} else {
|
|
this.stdErrorIndices = reader.int32s(this.ParametersCount);
|
|
this._coeffStdError = stdErrorValues;
|
|
}
|
|
this._bias = reader.float32();
|
|
const isWeightsDense = reader.byte();
|
|
const weightsLength = reader.int32();
|
|
const weightsValues = reader.float32s(weightsLength);
|
|
|
|
if (isWeightsDense) {
|
|
this._weights = weightsValues;
|
|
} else {
|
|
this.weightsIndices = reader.int32s(weightsLength);
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.LinearMulticlassModelParametersBase = class extends mlnet.ModelParametersBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
const numberOfFeatures = reader.int32();
|
|
const numberOfClasses = reader.int32();
|
|
this.Biases = reader.float32s(numberOfClasses);
|
|
const numStarts = reader.int32();
|
|
if (numStarts === 0) {
|
|
/* let numIndices = */ reader.int32();
|
|
/* let numWeights = */ reader.int32();
|
|
this.Weights = [];
|
|
for (let i = 0; i < numberOfClasses; i++) {
|
|
const w = reader.float32s(numberOfFeatures);
|
|
this.Weights.push(w);
|
|
}
|
|
} else {
|
|
|
|
const starts = reader.int32s(reader.int32());
|
|
/* let numIndices = */ reader.int32();
|
|
const indices = [];
|
|
for (let i = 0; i < numberOfClasses; i++) {
|
|
indices.push(reader.int32s(starts[i + 1] - starts[i]));
|
|
}
|
|
/* let numValues = */ reader.int32();
|
|
this.Weights = [];
|
|
for (let i = 0; i < numberOfClasses; i++) {
|
|
const values = reader.float32s(starts[i + 1] - starts[i]);
|
|
this.Weights.push(values);
|
|
}
|
|
}
|
|
|
|
const labelNamesReader = context.openBinary('LabelNames');
|
|
if (labelNamesReader) {
|
|
this.LabelNames = [];
|
|
for (let i = 0; i < numberOfClasses; i++) {
|
|
const id = labelNamesReader.int32();
|
|
this.LabelNames.push(context.strings[id]);
|
|
}
|
|
}
|
|
|
|
const statistics = context.open('ModelStats');
|
|
if (statistics) {
|
|
this.Statistics = statistics;
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.LinearMulticlassModelParameters = class extends mlnet.LinearMulticlassModelParametersBase {
|
|
};
|
|
|
|
mlnet.RegressionModelParameters = class extends mlnet.LinearModelParameters {
|
|
};
|
|
|
|
mlnet.PoissonRegressionModelParameters = class extends mlnet.RegressionModelParameters {
|
|
};
|
|
|
|
mlnet.LinearRegressionModelParameters = class extends mlnet.RegressionModelParameters {
|
|
};
|
|
|
|
mlnet.MaximumEntropyModelParameters = class extends mlnet.LinearMulticlassModelParametersBase {
|
|
};
|
|
|
|
mlnet.TokenizingByCharactersTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.UseMarkerChars = reader.boolean();
|
|
this.IsSeparatorStartEnd = context.modelVersionReadable < 0x00010002 ? true : reader.boolean();
|
|
}
|
|
};
|
|
|
|
mlnet.SequencePool = class {
|
|
|
|
constructor(reader) {
|
|
this.idLim = reader.int32();
|
|
this.start = reader.int32s(this.idLim + 1);
|
|
this.bytes = reader.read(this.start[this.idLim]);
|
|
}
|
|
};
|
|
|
|
mlnet.NgramExtractingTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
if (this.inputs.length === 1) {
|
|
this._option(context, reader, this);
|
|
} else {
|
|
// debugger;
|
|
}
|
|
}
|
|
|
|
_option(context, reader, option) {
|
|
const readWeighting = context.modelVersionReadable >= 0x00010002;
|
|
option.NgramLength = reader.int32();
|
|
option.SkipLength = reader.int32();
|
|
if (readWeighting) {
|
|
option.Weighting = reader.int32();
|
|
}
|
|
option.NonEmptyLevels = reader.booleans(option.NgramLength);
|
|
option.NgramMap = new mlnet.SequencePool(reader);
|
|
if (readWeighting) {
|
|
option.InvDocFreqs = reader.float64s(reader.int32());
|
|
}
|
|
}
|
|
};
|
|
|
|
// mlnet.NgramExtractingTransformer.WeightingCriteria
|
|
|
|
mlnet.NgramHashingTransformer = class extends mlnet.RowToRowTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const loadLegacy = context.modelVersionWritten < 0x00010003;
|
|
const reader = context.reader;
|
|
if (loadLegacy) {
|
|
reader.int32(); // cbFloat
|
|
}
|
|
this.inputs = [];
|
|
this.outputs = [];
|
|
const columnsLength = reader.int32();
|
|
if (loadLegacy) {
|
|
// for (let i = 0; i < columnsLength; i++) {
|
|
// this.Columns.push(new NgramHashingEstimator.ColumnOptions(context));
|
|
// }
|
|
} else {
|
|
for (let i = 0; i < columnsLength; i++) {
|
|
this.outputs.push(context.string());
|
|
const csrc = reader.int32();
|
|
for (let j = 0; j < csrc; j++) {
|
|
const src = context.string();
|
|
this.inputs.push(src);
|
|
// inputs[i][j] = src;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.WordTokenizingTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
if (this.inputs.length === 1) {
|
|
this.Separators = [];
|
|
const count = reader.int32();
|
|
for (let i = 0; i < count; i++) {
|
|
this.Separators.push(String.fromCharCode(reader.int16()));
|
|
}
|
|
} else {
|
|
// debugger;
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.TextNormalizingTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.CaseMode = reader.byte();
|
|
this.KeepDiacritics = reader.boolean();
|
|
this.KeepPunctuations = reader.boolean();
|
|
this.KeepNumbers = reader.boolean();
|
|
}
|
|
};
|
|
|
|
mlnet.TextNormalizingTransformer.CaseMode = {
|
|
Lower: 0,
|
|
Upper: 1,
|
|
None: 2
|
|
};
|
|
|
|
mlnet.PrincipalComponentAnalysisTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
if (context.modelVersionReadable === 0x00010001) {
|
|
if (reader.int32() !== 4) {
|
|
throw new mlnet.Error('This file was saved by an incompatible version.');
|
|
}
|
|
}
|
|
this.TransformInfos = [];
|
|
for (let i = 0; i < this.inputs.length; i++) {
|
|
const option = {};
|
|
option.Dimension = reader.int32();
|
|
option.Rank = reader.int32();
|
|
option.Eigenvectors = [];
|
|
for (let j = 0; j < option.Rank; j++) {
|
|
option.Eigenvectors.push(reader.float32s(option.Dimension));
|
|
}
|
|
option.MeanProjected = reader.float32s(reader.int32());
|
|
this.TransformInfos.push(option);
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.LpNormNormalizingTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
|
|
if (context.modelVersionWritten <= 0x00010002) {
|
|
/* cbFloat */ reader.int32();
|
|
}
|
|
// let normKindSerialized = context.modelVersionWritten >= 0x00010002;
|
|
if (this.inputs.length === 1) {
|
|
this.EnsureZeroMean = reader.boolean();
|
|
this.Norm = reader.byte();
|
|
this.Scale = reader.float32();
|
|
} else {
|
|
// debugger;
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.KeyToVectorMappingTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
if (context.modelVersionWritten === 0x00010001) {
|
|
/* cbFloat = */ reader.int32();
|
|
}
|
|
const columnsLength = this.inputs.length;
|
|
this.Bags = reader.booleans(columnsLength);
|
|
}
|
|
};
|
|
|
|
mlnet.TypeConvertingTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
};
|
|
|
|
mlnet.ImageLoadingTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
this.ImageFolder = context.string(null);
|
|
}
|
|
};
|
|
|
|
mlnet.ImageResizingTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
if (this.inputs.length === 1) {
|
|
this._option(reader, this);
|
|
} else {
|
|
this.Options = [];
|
|
for (let i = 0; i < this.inputs.length; i++) {
|
|
const option = {};
|
|
this._option(reader, option);
|
|
this.Options.push(option);
|
|
}
|
|
}
|
|
}
|
|
|
|
_option(reader, option) {
|
|
option.Width = reader.int32();
|
|
option.Height = reader.int32();
|
|
option.Resizing = reader.byte();
|
|
option.Anchor = reader.byte();
|
|
}
|
|
};
|
|
|
|
mlnet.ImageResizingTransformer.ResizingKind = {
|
|
IsoPad: 0,
|
|
IsoCrop: 1,
|
|
Fill: 2
|
|
};
|
|
|
|
mlnet.ImageResizingTransformer.Anchor = {
|
|
Right: 0,
|
|
Left: 1,
|
|
Top: 2,
|
|
Bottom: 3,
|
|
Center: 4
|
|
};
|
|
|
|
mlnet.ImagePixelExtractingTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
if (this.inputs.length === 1) {
|
|
this._option(context, reader, this);
|
|
} else {
|
|
this.Options = [];
|
|
for (let i = 0; i < this.inputs.length; i++) {
|
|
const option = {};
|
|
this._option(context, reader, option);
|
|
this.Options.push(option);
|
|
}
|
|
}
|
|
}
|
|
|
|
_option(context, reader, option) {
|
|
option.ColorsToExtract = reader.byte();
|
|
option.OrderOfExtraction = context.modelVersionWritten <= 0x00010002 ? mlnet.ImagePixelExtractingTransformer.ColorsOrder.ARGB : reader.byte();
|
|
let planes = option.ColorsToExtract;
|
|
planes = (planes & 0x05) + ((planes >> 1) & 0x05);
|
|
planes = (planes & 0x03) + ((planes >> 2) & 0x03);
|
|
option.Planes = planes & 0xFF;
|
|
option.OutputAsFloatArray = reader.boolean();
|
|
option.OffsetImage = reader.float32();
|
|
option.ScaleImage = reader.float32();
|
|
option.InterleavePixelColors = reader.boolean();
|
|
}
|
|
};
|
|
|
|
mlnet.ImagePixelExtractingTransformer.ColorBits = {
|
|
Alpha: 0x01,
|
|
Red: 0x02,
|
|
Green: 0x04,
|
|
Blue: 0x08,
|
|
Rgb: 0x0E,
|
|
All: 0x0F
|
|
};
|
|
|
|
mlnet.ImagePixelExtractingTransformer.ColorsOrder = {
|
|
ARGB: 1,
|
|
ARBG: 2,
|
|
ABRG: 3,
|
|
ABGR: 4,
|
|
AGRB: 5,
|
|
AGBR: 6
|
|
};
|
|
|
|
mlnet.NormalizingTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.Options = [];
|
|
for (let i = 0; i < this.inputs.length; i++) {
|
|
let isVector = false;
|
|
let shape = 0;
|
|
let itemKind = '';
|
|
if (context.modelVersionWritten < 0x00010002) {
|
|
isVector = reader.boolean();
|
|
shape = [reader.int32()];
|
|
itemKind = reader.byte();
|
|
} else {
|
|
isVector = reader.boolean();
|
|
itemKind = reader.byte();
|
|
shape = reader.int32s(reader.int32());
|
|
}
|
|
let itemType = '';
|
|
switch (itemKind) {
|
|
case 9: itemType = 'float32'; break;
|
|
case 10: itemType = 'float64'; break;
|
|
default: throw new mlnet.Error(`Unsupported NormalizingTransformer item kind '${itemKind}'.`);
|
|
}
|
|
const type = itemType + (isVector ? `[${shape.map((dim) => dim.toString()).join(',')}]` : '');
|
|
const name = `Normalizer_${(`00${i}`).slice(-3)}`;
|
|
const func = context.open(name);
|
|
this.Options.push({ type, func });
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.KeyToValueMappingTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
};
|
|
|
|
mlnet.ValueToKeyMappingTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
if (context.modelVersionWritten >= 0x00010003) {
|
|
this.textMetadata = reader.booleans(this.outputs.length + this.inputs.length);
|
|
} else {
|
|
this.textMetadata = [];
|
|
for (let i = 0; i < this.columnPairs.length; i++) {
|
|
this.textMetadata.push(false);
|
|
}
|
|
}
|
|
const vocabulary = context.open('Vocabulary');
|
|
if (vocabulary) {
|
|
this.termMap = vocabulary.termMap;
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.TermMap = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
const mtype = reader.byte();
|
|
switch (mtype) {
|
|
case 0: { // Text
|
|
this.values = [];
|
|
const cstr = reader.int32();
|
|
for (let i = 0; i < cstr; i++) {
|
|
this.values.push(context.string());
|
|
}
|
|
break;
|
|
}
|
|
case 1: { // Codec
|
|
const codec = new mlnet.Codec(reader);
|
|
const count = reader.int32();
|
|
this.values = codec.read(reader, count);
|
|
break;
|
|
}
|
|
default:
|
|
throw new mlnet.Error(`Unsupported term map type '${mtype}'.`);
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.TermManager = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
const cmap = reader.int32();
|
|
this.termMap = [];
|
|
if (context.modelVersionWritten >= 0x00010002) {
|
|
for (let i = 0; i < cmap; ++i) {
|
|
this.termMap.push(new mlnet.TermMap(context));
|
|
// debugger;
|
|
// termMap[i] = TermMap.Load(c, host, CodecFactory);
|
|
}
|
|
} else {
|
|
throw new mlnet.Error('Unsupported TermManager version.');
|
|
// for (let i = 0; i < cmap; ++i) {
|
|
// debugger;
|
|
// // termMap[i] = TermMap.TextImpl.Create(c, host)
|
|
// }
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.ValueMappingTransformer = class extends mlnet.OneToOneTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
this.keyColumnName = 'Key';
|
|
if (context.check('TXTLOOKT', 0x00010002, 0x00010002)) {
|
|
this.keyColumnName = 'Term';
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.KeyToVectorTransform = class {
|
|
};
|
|
|
|
mlnet.GenericScoreTransform = class {
|
|
};
|
|
|
|
mlnet.CompositeDataLoader = class {
|
|
|
|
constructor(context) {
|
|
/* let loader = */ context.open('Loader');
|
|
const reader = context.reader;
|
|
// LoadTransforms
|
|
reader.int32(); // floatSize
|
|
const cxf = reader.int32();
|
|
const tagData = [];
|
|
for (let i = 0; i < cxf; i++) {
|
|
let tag = '';
|
|
let args = null;
|
|
if (context.modelVersionReadable >= 0x00010002) {
|
|
tag = context.string();
|
|
args = context.string(null);
|
|
}
|
|
tagData.push([tag, args]);
|
|
}
|
|
this.chain = [];
|
|
for (let j = 0; j < cxf; j++) {
|
|
const name = `Transform_${(`00${j}`).slice(-3)}`;
|
|
const transform = context.open(name);
|
|
this.chain.push(transform);
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.RowToRowMapperTransform = class extends mlnet.RowToRowTransformBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const mapper = context.open('Mapper');
|
|
this.__type__ = mapper.__type__;
|
|
for (const key of Object.keys(mapper)) {
|
|
this[key] = mapper[key];
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.ImageClassificationTransformer = class extends mlnet.RowToRowTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.addBatchDimensionInput = reader.boolean();
|
|
const numInputs = reader.int32();
|
|
this.inputs = [];
|
|
for (let i = 0; i < numInputs; i++) {
|
|
this.inputs.push({ name: context.string() });
|
|
}
|
|
this.outputs = [];
|
|
const numOutputs = reader.int32();
|
|
for (let i = 0; i < numOutputs; i++) {
|
|
this.outputs.push({ name: context.string() });
|
|
}
|
|
this.labelColumn = reader.string();
|
|
this.checkpointName = reader.string();
|
|
this.arch = reader.int32(); // Architecture
|
|
this.scoreColumnName = reader.string();
|
|
this.predictedColumnName = reader.string();
|
|
this.learningRate = reader.float32();
|
|
this.classCount = reader.int32();
|
|
this.keyValueAnnotations = [];
|
|
for (let i = 0; i < this.classCount; i++) {
|
|
this.keyValueAnnotations.push(context.string());
|
|
}
|
|
this.predictionTensorName = reader.string();
|
|
this.softMaxTensorName = reader.string();
|
|
this.jpegDataTensorName = reader.string();
|
|
this.resizeTensorName = reader.string();
|
|
}
|
|
};
|
|
|
|
mlnet.OnnxTransformer = class extends mlnet.RowToRowTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
const modelReader = context.openBinary('OnnxModel');
|
|
if (modelReader) {
|
|
const size = modelReader.uint32();
|
|
context.resolve(this, 'onnx', modelReader.read(size));
|
|
}
|
|
const numInputs = context.modelVersionWritten > 0x00010001 ? reader.int32() : 1;
|
|
this.inputs = [];
|
|
for (let i = 0; i < numInputs; i++) {
|
|
this.inputs.push({ name: context.string() });
|
|
}
|
|
const numOutputs = context.modelVersionWritten > 0x00010001 ? reader.int32() : 1;
|
|
this.outputs = [];
|
|
for (let i = 0; i < numOutputs; i++) {
|
|
this.outputs.push({ name: context.string() });
|
|
}
|
|
if (context.modelVersionWritten > 0x0001000C) {
|
|
const customShapeInfosLength = reader.int32();
|
|
this.LoadedCustomShapeInfos = [];
|
|
for (let i = 0; i < customShapeInfosLength; i++) {
|
|
this.LoadedCustomShapeInfos.push({
|
|
name: context.string(),
|
|
shape: reader.int32s(reader.int32())
|
|
});
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.OptionalColumnTransform = class extends mlnet.RowToRowMapperTransformBase {
|
|
};
|
|
|
|
mlnet.TensorFlowTransformer = class extends mlnet.RowToRowTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.IsFrozen = context.modelVersionReadable >= 0x00010002 ? reader.boolean() : true;
|
|
this.AddBatchDimensionInput = context.modelVersionReadable >= 0x00010003 ? reader.boolean() : true;
|
|
const numInputs = reader.int32();
|
|
this.inputs = [];
|
|
for (let i = 0; i < numInputs; i++) {
|
|
this.inputs.push({ name: context.string() });
|
|
}
|
|
const numOutputs = context.modelVersionReadable >= 0x00010002 ? reader.int32() : 1;
|
|
this.outputs = [];
|
|
for (let i = 0; i < numOutputs; i++) {
|
|
this.outputs.push({ name: context.string() });
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.OneVersusAllModelParameters = class extends mlnet.ModelParametersBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.UseDist = reader.boolean();
|
|
const len = reader.int32();
|
|
this.chain = [];
|
|
for (let i = 0; i < len; i++) {
|
|
const name = `SubPredictor_${(`00${i}`).slice(-3)}`;
|
|
const predictor = context.open(name);
|
|
this.chain.push(predictor);
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.TextFeaturizingEstimator = class {
|
|
|
|
constructor(context) {
|
|
|
|
if (context.modelVersionReadable === 0x00010001) {
|
|
const reader = context.reader;
|
|
const n = reader.int32();
|
|
this.chain = [];
|
|
/* let loader = */ context.open('Loader');
|
|
for (let i = 0; i < n; i++) {
|
|
const name = `Step_${(`00${i}`).slice(-3)}`;
|
|
const transformer = context.open(name);
|
|
this.chain.push(transformer);
|
|
// debugger;
|
|
}
|
|
|
|
// throw new mlnet.Error('Unsupported TextFeaturizingEstimator format.');
|
|
} else {
|
|
const chain = context.open('Chain');
|
|
this.chain = chain.chain;
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.TextLoader = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
reader.int32(); // floatSize
|
|
this.MaxRows = reader.int64();
|
|
this.Flags = reader.uint32();
|
|
this.InputSize = reader.int32();
|
|
const separatorCount = reader.int32();
|
|
this.Separators = [];
|
|
for (let i = 0; i < separatorCount; i++) {
|
|
this.Separators.push(String.fromCharCode(reader.uint16()));
|
|
}
|
|
this.Bindinds = new mlnet.TextLoader.Bindinds(context);
|
|
}
|
|
};
|
|
|
|
mlnet.TextLoader.Bindinds = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
const cinfo = reader.int32();
|
|
for (let i = 0; i < cinfo; i++) {
|
|
// debugger;
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.CalibratedPredictorBase = class {
|
|
|
|
constructor(predictor, calibrator) {
|
|
this.SubPredictor = predictor;
|
|
this.Calibrator = calibrator;
|
|
}
|
|
};
|
|
|
|
mlnet.ValueMapperCalibratedPredictorBase = class extends mlnet.CalibratedPredictorBase {
|
|
};
|
|
|
|
mlnet.CalibratedModelParametersBase = class {
|
|
|
|
constructor(context) {
|
|
this.Predictor = context.open('Predictor');
|
|
this.Calibrator = context.open('Calibrator');
|
|
}
|
|
};
|
|
|
|
mlnet.ValueMapperCalibratedModelParametersBase = class extends mlnet.CalibratedModelParametersBase {
|
|
};
|
|
|
|
mlnet.CalibratedPredictor = class extends mlnet.ValueMapperCalibratedPredictorBase {
|
|
|
|
constructor(context) {
|
|
const predictor = context.open('Predictor');
|
|
const calibrator = context.open('Calibrator');
|
|
super(predictor, calibrator);
|
|
}
|
|
};
|
|
|
|
mlnet.ParameterMixingCalibratedModelParameters = class extends mlnet.ValueMapperCalibratedModelParametersBase {
|
|
};
|
|
|
|
mlnet.FieldAwareFactorizationMachineModelParameters = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
this.Norm = reader.boolean();
|
|
this.FieldCount = reader.int32();
|
|
this.FeatureCount = reader.int32();
|
|
this.LatentDim = reader.int32();
|
|
this.LinearWeights = reader.float32s(reader.int32());
|
|
this.LatentWeights = reader.float32s(reader.int32());
|
|
}
|
|
};
|
|
|
|
mlnet.KMeansModelParameters = class extends mlnet.ModelParametersBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.k = reader.int32();
|
|
this.Dimensionality = reader.int32();
|
|
this.Centroids = [];
|
|
for (let i = 0; i < this.k; i++) {
|
|
const count = context.modelVersionWritten >= 0x00010002 ? reader.int32() : this.Dimensionality;
|
|
const indices = count < this.Dimensionality ? reader.int32s(count) : null;
|
|
const values = reader.float32s(count);
|
|
this.Centroids.push({ indices, values });
|
|
}
|
|
// input type = float32[dimensionality]
|
|
// output type = float32[k]
|
|
}
|
|
};
|
|
|
|
mlnet.PcaModelParameters = class extends mlnet.ModelParametersBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.Dimension = reader.int32();
|
|
this.Rank = reader.int32();
|
|
const center = reader.boolean();
|
|
if (center) {
|
|
this.Mean = reader.float32s(this.Dimension);
|
|
} else {
|
|
this.Mean = [];
|
|
}
|
|
this.EigenVectors = [];
|
|
for (let i = 0; i < this.Rank; ++i) {
|
|
this.EigenVectors.push(reader.float32s(this.Dimension));
|
|
}
|
|
// input type -> float32[Dimension]
|
|
}
|
|
};
|
|
|
|
mlnet.TreeEnsembleModelParameters = class extends mlnet.ModelParametersBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
const usingDefaultValues = context.modelVersionWritten >= this.VerDefaultValueSerialized;
|
|
const categoricalSplits = context.modelVersionWritten >= this.VerCategoricalSplitSerialized;
|
|
this.TrainedEnsemble = new mlnet.InternalTreeEnsemble(context, usingDefaultValues, categoricalSplits);
|
|
this.InnerOptions = context.string(null);
|
|
if (context.modelVersionWritten >= this.VerNumFeaturesSerialized) {
|
|
this.NumFeatures = reader.int32();
|
|
}
|
|
// input type -> float32[NumFeatures]
|
|
// output type -> float32
|
|
}
|
|
};
|
|
|
|
mlnet.InternalTreeEnsemble = class {
|
|
|
|
constructor(context, usingDefaultValues, categoricalSplits) {
|
|
const reader = context.reader;
|
|
this.Trees = [];
|
|
const numTrees = reader.int32();
|
|
for (let i = 0; i < numTrees; i++) {
|
|
switch (reader.byte()) {
|
|
case mlnet.InternalTreeEnsemble.TreeType.Regression:
|
|
this.Trees.push(new mlnet.InternalRegressionTree(context, usingDefaultValues, categoricalSplits));
|
|
break;
|
|
case mlnet.InternalTreeEnsemble.TreeType.FastForest:
|
|
this.Trees.push(new mlnet.InternalQuantileRegressionTree(context, usingDefaultValues, categoricalSplits));
|
|
break;
|
|
case mlnet.InternalTreeEnsemble.TreeType.Affine:
|
|
// Affine regression trees do not actually work, nor is it clear how they ever
|
|
// could have worked within TLC, so the chance of this happening seems remote.
|
|
throw new mlnet.Error('Affine regression trees unsupported.');
|
|
default:
|
|
throw new mlnet.Error('Unsupported ensemble tree type.');
|
|
}
|
|
}
|
|
this.Bias = reader.float64();
|
|
this.FirstInputInitializationContent = context.string(null);
|
|
}
|
|
};
|
|
|
|
mlnet.InternalRegressionTree = class {
|
|
|
|
constructor(context, usingDefaultValue, categoricalSplits) {
|
|
const reader = context.reader;
|
|
this.NumLeaves = reader.int32();
|
|
this.MaxOuptut = reader.float64();
|
|
this.Weight = reader.float64();
|
|
this.LteChild = reader.int32s(reader.int32());
|
|
this.GtChild = reader.int32s(reader.int32());
|
|
this.SplitFeatures = reader.int32s(reader.int32());
|
|
if (categoricalSplits) {
|
|
const categoricalNodeIndices = reader.int32s(reader.int32());
|
|
if (categoricalNodeIndices.length > 0) {
|
|
this.CategoricalSplitFeatures = [];
|
|
this.CategoricalSplitFeatureRanges = [];
|
|
for (const index of categoricalNodeIndices) {
|
|
this.CategoricalSplitFeatures[index] = reader.int32s(reader.int32());
|
|
this.CategoricalSplitFeatureRanges[index] = reader.int32s(2);
|
|
}
|
|
}
|
|
}
|
|
this.Thresholds = reader.uint32s(reader.int32());
|
|
this.RawThresholds = reader.float32s(reader.int32());
|
|
this.DefaultValueForMissing = usingDefaultValue ? reader.float32s(reader.int32()) : null;
|
|
this.LeafValues = reader.float64s(reader.int32());
|
|
|
|
this.SplitGain = reader.float64s(reader.int32());
|
|
this.GainPValue = reader.float64s(reader.int32());
|
|
this.PreviousLeafValue = reader.float64s(reader.int32());
|
|
}
|
|
};
|
|
|
|
mlnet.InternalTreeEnsemble.TreeType = {
|
|
Regression: 0,
|
|
Affine: 1,
|
|
FastForest: 2
|
|
};
|
|
|
|
mlnet.TreeEnsembleModelParametersBasedOnRegressionTree = class extends mlnet.TreeEnsembleModelParameters {
|
|
};
|
|
|
|
mlnet.FastTreeTweedieModelParameters = class extends mlnet.TreeEnsembleModelParametersBasedOnRegressionTree {
|
|
|
|
get VerNumFeaturesSerialized() {
|
|
return 0x00010001;
|
|
}
|
|
|
|
get VerDefaultValueSerialized() {
|
|
return 0x00010002;
|
|
}
|
|
|
|
get VerCategoricalSplitSerialized() {
|
|
return 0x00010003;
|
|
}
|
|
};
|
|
|
|
mlnet.FastTreeRankingModelParameters = class extends mlnet.TreeEnsembleModelParametersBasedOnRegressionTree {
|
|
|
|
get VerNumFeaturesSerialized() {
|
|
return 0x00010002;
|
|
}
|
|
|
|
get VerDefaultValueSerialized() {
|
|
return 0x00010004;
|
|
}
|
|
|
|
get VerCategoricalSplitSerialized() {
|
|
return 0x00010005;
|
|
}
|
|
};
|
|
|
|
mlnet.FastTreeBinaryModelParameters = class extends mlnet.TreeEnsembleModelParametersBasedOnRegressionTree {
|
|
|
|
get VerNumFeaturesSerialized() {
|
|
return 0x00010002;
|
|
}
|
|
|
|
get VerDefaultValueSerialized() {
|
|
return 0x00010004;
|
|
}
|
|
|
|
get VerCategoricalSplitSerialized() {
|
|
return 0x00010005;
|
|
}
|
|
};
|
|
|
|
mlnet.FastTreeRegressionModelParameters = class extends mlnet.TreeEnsembleModelParametersBasedOnRegressionTree {
|
|
|
|
get VerNumFeaturesSerialized() {
|
|
return 0x00010002;
|
|
}
|
|
|
|
get VerDefaultValueSerialized() {
|
|
return 0x00010004;
|
|
}
|
|
|
|
get VerCategoricalSplitSerialized() {
|
|
return 0x00010005;
|
|
}
|
|
};
|
|
|
|
mlnet.LightGbmRegressionModelParameters = class extends mlnet.TreeEnsembleModelParametersBasedOnRegressionTree {
|
|
|
|
get VerNumFeaturesSerialized() {
|
|
return 0x00010002;
|
|
}
|
|
|
|
get VerDefaultValueSerialized() {
|
|
return 0x00010004;
|
|
}
|
|
|
|
get VerCategoricalSplitSerialized() {
|
|
return 0x00010005;
|
|
}
|
|
};
|
|
|
|
mlnet.LightGbmBinaryModelParameters = class extends mlnet.TreeEnsembleModelParametersBasedOnRegressionTree {
|
|
|
|
get VerNumFeaturesSerialized() {
|
|
return 0x00010002;
|
|
}
|
|
|
|
get VerDefaultValueSerialized() {
|
|
return 0x00010004;
|
|
}
|
|
|
|
get VerCategoricalSplitSerialized() {
|
|
return 0x00010005;
|
|
}
|
|
};
|
|
|
|
mlnet.FeatureWeightsCalibratedModelParameters = class extends mlnet.ValueMapperCalibratedModelParametersBase {
|
|
};
|
|
|
|
mlnet.FastTreePredictionWrapper = class {
|
|
};
|
|
|
|
mlnet.FastForestClassificationPredictor = class extends mlnet.FastTreePredictionWrapper {
|
|
};
|
|
|
|
mlnet.PlattCalibrator = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
this.ParamA = reader.float64();
|
|
this.ParamB = reader.float64();
|
|
}
|
|
};
|
|
|
|
mlnet.Codec = class {
|
|
|
|
constructor(reader) {
|
|
this.name = reader.string();
|
|
const size = reader.leb128();
|
|
const data = reader.read(size);
|
|
reader = new mlnet.BinaryReader(data);
|
|
switch (this.name) {
|
|
case 'Boolean': break;
|
|
case 'Single': break;
|
|
case 'Double': break;
|
|
case 'Byte': break;
|
|
case 'Int32': break;
|
|
case 'UInt32': break;
|
|
case 'Int64': break;
|
|
case 'TextSpan': break;
|
|
case 'VBuffer':
|
|
this.itemType = new mlnet.Codec(reader);
|
|
this.dims = reader.int32s(reader.int32());
|
|
break;
|
|
case 'Key':
|
|
case 'Key2':
|
|
this.itemType = new mlnet.Codec(reader);
|
|
this.count = reader.uint64().toNumber();
|
|
break;
|
|
default:
|
|
throw new mlnet.Error(`Unsupported codec '${this.name}'.`);
|
|
}
|
|
}
|
|
|
|
read(reader, count) {
|
|
const values = [];
|
|
switch (this.name) {
|
|
case 'Single':
|
|
for (let i = 0; i < count; i++) {
|
|
values.push(reader.float32());
|
|
}
|
|
break;
|
|
case 'Int32':
|
|
for (let i = 0; i < count; i++) {
|
|
values.push(reader.int32());
|
|
}
|
|
break;
|
|
case 'Int64':
|
|
for (let i = 0; i < count; i++) {
|
|
values.push(reader.int64());
|
|
}
|
|
break;
|
|
default:
|
|
throw new mlnet.Error(`Unsupported codec read operation '${this.name}'.`);
|
|
}
|
|
return values;
|
|
}
|
|
};
|
|
|
|
mlnet.SequentialTransformerBase = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
this.WindowSize = reader.int32();
|
|
this.InitialWindowSize = reader.int32();
|
|
this.inputs = [];
|
|
this.inputs.push({ name: context.string() });
|
|
this.outputs = [];
|
|
this.outputs.push({ name: context.string() });
|
|
this.ConfidenceLowerBoundColumn = reader.string();
|
|
this.ConfidenceUpperBoundColumn = reader.string();
|
|
this.Type = new mlnet.Codec(reader);
|
|
}
|
|
};
|
|
|
|
mlnet.AnomalyDetectionStateBase = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
this.LogMartingaleUpdateBuffer = mlnet.AnomalyDetectionStateBase._deserializeFixedSizeQueueDouble(reader);
|
|
this.RawScoreBuffer = mlnet.AnomalyDetectionStateBase._deserializeFixedSizeQueueDouble(reader);
|
|
this.LogMartingaleValue = reader.float64();
|
|
this.SumSquaredDist = reader.float64();
|
|
this.MartingaleAlertCounter = reader.int32();
|
|
}
|
|
|
|
static _deserializeFixedSizeQueueDouble(reader) {
|
|
/* let capacity = */ reader.int32();
|
|
const count = reader.int32();
|
|
const queue = [];
|
|
for (let i = 0; i < count; i++) {
|
|
queue.push(reader.float64());
|
|
}
|
|
return queue;
|
|
}
|
|
};
|
|
|
|
mlnet.SequentialAnomalyDetectionTransformBase = class extends mlnet.SequentialTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.Martingale = reader.byte();
|
|
this.ThresholdScore = reader.byte();
|
|
this.Side = reader.byte();
|
|
this.PowerMartingaleEpsilon = reader.float64();
|
|
this.AlertThreshold = reader.float64();
|
|
this.State = new mlnet.AnomalyDetectionStateBase(context);
|
|
}
|
|
};
|
|
|
|
mlnet.TimeSeriesUtils = class {
|
|
|
|
static deserializeFixedSizeQueueSingle(reader) {
|
|
/* const capacity = */ reader.int32();
|
|
const count = reader.int32();
|
|
const queue = [];
|
|
for (let i = 0; i < count; i++) {
|
|
queue.push(reader.float32());
|
|
}
|
|
return queue;
|
|
}
|
|
};
|
|
|
|
mlnet.IidAnomalyDetectionBase = class extends mlnet.SequentialAnomalyDetectionTransformBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.WindowedBuffer = mlnet.TimeSeriesUtils.deserializeFixedSizeQueueSingle(reader);
|
|
this.InitialWindowedBuffer = mlnet.TimeSeriesUtils.deserializeFixedSizeQueueSingle(reader);
|
|
}
|
|
};
|
|
|
|
mlnet.IidAnomalyDetectionBaseWrapper = class {
|
|
|
|
constructor(context) {
|
|
const internalTransform = new mlnet.IidAnomalyDetectionBase(context);
|
|
for (const key of Object.keys(internalTransform)) {
|
|
this[key] = internalTransform[key];
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.IidChangePointDetector = class extends mlnet.IidAnomalyDetectionBaseWrapper {
|
|
};
|
|
|
|
mlnet.IidSpikeDetector = class extends mlnet.IidAnomalyDetectionBaseWrapper {
|
|
};
|
|
|
|
mlnet.SequenceModelerBase = class {
|
|
};
|
|
|
|
mlnet.RankSelectionMethod = {
|
|
Fixed: 0,
|
|
Exact: 1,
|
|
Fact: 2
|
|
};
|
|
|
|
mlnet.AdaptiveSingularSpectrumSequenceModelerInternal = class extends mlnet.SequenceModelerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this._seriesLength = reader.int32();
|
|
this._windowSize = reader.int32();
|
|
this._trainSize = reader.int32();
|
|
this._rank = reader.int32();
|
|
this._discountFactor = reader.float32();
|
|
this._rankSelectionMethod = reader.byte(); // RankSelectionMethod
|
|
const isWeightSet = reader.byte();
|
|
this._alpha = reader.float32s(reader.int32());
|
|
if (context.modelVersionReadable >= 0x00010002) {
|
|
this._state = reader.float32s(reader.int32());
|
|
}
|
|
this.ShouldComputeForecastIntervals = reader.byte();
|
|
this._observationNoiseVariance = reader.float32();
|
|
this._autoregressionNoiseVariance = reader.float32();
|
|
this._observationNoiseMean = reader.float32();
|
|
this._autoregressionNoiseMean = reader.float32();
|
|
if (context.modelVersionReadable >= 0x00010002) {
|
|
this._nextPrediction = reader.float32();
|
|
}
|
|
this._maxRank = reader.int32();
|
|
this._shouldStablize = reader.byte();
|
|
this._shouldMaintainInfo = reader.byte();
|
|
this._maxTrendRatio = reader.float64();
|
|
if (isWeightSet) {
|
|
this._wTrans = reader.float32s(reader.int32());
|
|
this._y = reader.float32s(reader.int32());
|
|
}
|
|
this._buffer = mlnet.TimeSeriesUtils.deserializeFixedSizeQueueSingle(reader);
|
|
}
|
|
};
|
|
|
|
mlnet.SequentialForecastingTransformBase = class extends mlnet.SequentialTransformerBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this._outputLength = reader.int32();
|
|
}
|
|
};
|
|
|
|
mlnet.SsaForecastingBaseWrapper = class extends mlnet.SequentialForecastingTransformBase {
|
|
|
|
constructor(context) {
|
|
super(context);
|
|
const reader = context.reader;
|
|
this.IsAdaptive = reader.boolean();
|
|
this.Horizon = reader.int32();
|
|
this.ConfidenceLevel = reader.float32();
|
|
this.WindowedBuffer = mlnet.TimeSeriesUtils.deserializeFixedSizeQueueSingle(reader);
|
|
this.InitialWindowedBuffer = mlnet.TimeSeriesUtils.deserializeFixedSizeQueueSingle(reader);
|
|
this.Model = context.open('SSA');
|
|
}
|
|
};
|
|
|
|
mlnet.SsaForecastingTransformer = class extends mlnet.SsaForecastingBaseWrapper {
|
|
};
|
|
|
|
mlnet.ColumnSelectingTransformer = class {
|
|
|
|
constructor(context) {
|
|
const reader = context.reader;
|
|
if (context.check('DRPCOLST', 0x00010002, 0x00010002)) {
|
|
throw new mlnet.Error("'LoadDropColumnsTransform' not supported.");
|
|
} else if (context.check('CHSCOLSF', 0x00010001, 0x00010001)) {
|
|
reader.int32(); // cbFloat
|
|
this.KeepHidden = this._getHiddenOption(reader.byte());
|
|
const count = reader.int32();
|
|
this.inputs = [];
|
|
for (let colIdx = 0; colIdx < count; colIdx++) {
|
|
const dst = context.string();
|
|
this.inputs.push(dst);
|
|
context.string(); // src
|
|
this._getHiddenOption(reader.byte()); // colKeepHidden
|
|
}
|
|
} else {
|
|
const keepColumns = reader.boolean();
|
|
this.KeepHidden = reader.boolean();
|
|
this.IgnoreMissing = reader.boolean();
|
|
const length = reader.int32();
|
|
this.inputs = [];
|
|
for (let i = 0; i < length; i++) {
|
|
this.inputs.push({ name: context.string() });
|
|
}
|
|
if (keepColumns) {
|
|
this.ColumnsToKeep = this.inputs;
|
|
} else {
|
|
this.ColumnsToDrop = this.inputs;
|
|
}
|
|
}
|
|
}
|
|
|
|
_getHiddenOption(value) {
|
|
switch (value) {
|
|
case 1: return true;
|
|
case 2: return false;
|
|
default: throw new mlnet.Error('Unsupported hide option specified');
|
|
}
|
|
}
|
|
};
|
|
|
|
mlnet.XGBoostMulticlass = class {};
|
|
|
|
mlnet.NltTokenizeTransform = class {};
|
|
|
|
mlnet.DropColumnsTransform = class {};
|
|
|
|
mlnet.StopWordsTransform = class {};
|
|
|
|
mlnet.CSharpTransform = class {};
|
|
|
|
mlnet.GenericScoreTransform = class {};
|
|
|
|
mlnet.NormalizeTransform = class {};
|
|
|
|
mlnet.CdfColumnFunction = class {
|
|
};
|
|
|
|
mlnet.MultiClassNetPredictor = class {};
|
|
|
|
mlnet.ProtonNNMCPred = class {};
|
|
|
|
mlnet.Utility = class {
|
|
|
|
static enum(type, value) {
|
|
if (type) {
|
|
mlnet.Utility._enums = mlnet.Utility._enums || new Map();
|
|
if (!mlnet.Utility._enums.has(type)) {
|
|
let obj = mlnet;
|
|
const id = type.split('.');
|
|
while (obj && id.length > 0) {
|
|
obj = obj[id.shift()];
|
|
}
|
|
if (obj) {
|
|
const entries = new Map(Object.entries(obj).map(([key, value]) => [value, key]));
|
|
mlnet.Utility._enums.set(type, entries);
|
|
} else {
|
|
mlnet.Utility._enums.set(type, new Map());
|
|
}
|
|
}
|
|
const map = mlnet.Utility._enums.get(type);
|
|
if (map.has(value)) {
|
|
return map.get(value);
|
|
}
|
|
}
|
|
return value;
|
|
}
|
|
};
|
|
|
|
mlnet.Error = class extends Error {
|
|
|
|
constructor(message) {
|
|
super(message);
|
|
this.name = 'Error loading ML.NET model.';
|
|
}
|
|
};
|
|
|
|
export const ModelFactory = mlnet.ModelFactory;
|