Files
wehub-resource-sync 7254f7b4d1
Build / Build (macos-latest) (push) Has been cancelled
Build / Build (ubuntu-latest) (push) Has been cancelled
Build / Build (windows-latest) (push) Has been cancelled
Build / Analyze (javascript) (push) Has been cancelled
Build / Analyze (python) (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 12:37:45 +08:00

97 lines
2.9 KiB
JavaScript

const sentencepiece = {};
sentencepiece.ModelFactory = class {
async match(context) {
const tags = await context.tags('pb');
if ((tags.size >= 3 && tags.size <= 5 &&
tags.get(1) === 2 && tags.get(2) === 2 && tags.get(3) === 2) &&
Array.from(tags).every(([key, value]) => (key <= 5 && value === 2))) {
const model = await context.tags('pb+');
if (model &&
model['1'] && model['1']['1'] === 2 && model['1']['2'] === 5 && model['1']['3'] === 0 &&
model['2'] && model['2']['4'] === 0) {
return context.set('sentencepiece');
}
}
return null;
}
async open(context) {
sentencepiece.proto = await context.require('./sentencepiece-proto');
sentencepiece.proto = sentencepiece.proto.sentencepiece;
let model = null;
try {
const reader = await context.read('protobuf.binary');
model = sentencepiece.proto.ModelProto.decode(reader);
} catch (error) {
const message = error && error.message ? error.message : error.toString();
throw new sentencepiece.Error(`File format is not sentencepiece.ModelProto (${message.replace(/\.$/, '')}).`);
}
return new sentencepiece.Model(model);
}
};
sentencepiece.Model = class {
constructor(model) {
this.format = 'SentencePiece';
this.modules = [new sentencepiece.Graph(model)];
}
};
sentencepiece.Graph = class {
constructor(model) {
this.inputs = [];
this.outputs = [];
this.nodes = [];
for (const [name, value] of Object.entries(model)) {
const node = new sentencepiece.Node(name, value);
this.nodes.push(node);
}
}
};
sentencepiece.Argument = class {
constructor(name, value) {
this.name = name;
this.value = value;
}
};
sentencepiece.Node = class {
constructor(name, obj) {
this.name = name;
this.inputs = [];
this.outputs = [];
this.attributes = [];
if (Array.isArray(obj)) {
const type = new Set(obj.map((value) => value.constructor.name));
this.type = { name: `${Array.from(type)[0]}[]` };
const attribute = new sentencepiece.Argument(name, obj);
this.attributes.push(attribute);
} else {
this.type = { name: obj.constructor.name };
for (const [name, value] of Object.entries(obj)) {
const data = ArrayBuffer.isView(value) ? Array.from(value) : value;
const attribute = new sentencepiece.Argument(name, data);
this.attributes.push(attribute);
}
}
}
};
sentencepiece.Error = class extends Error {
constructor(message) {
super(message);
this.name = 'Error loading SentencePiece model.';
}
};
export const ModelFactory = sentencepiece.ModelFactory;